jexidb 2.1.2 → 2.1.4

package/src/Database.mjs

@@ -770,10 +770,19 @@ class Database extends EventEmitter {
       }
       
       // Manual save is now the default behavior
-      
+
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length)
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`)
+        }
+      }
+
       this.initialized = true
       this.emit('initialized')
-      
+
       if (this.opts.debugMode) {
         console.log(`✅ Database initialized with ${this.writeBuffer.length} records`)
       }
@@ -1319,38 +1328,21 @@ class Database extends EventEmitter {
         this.pendingIndexUpdates = []
       }
       
-      // CRITICAL FIX: Flush write buffer completely after capturing snapshot
-      await this._flushWriteBufferCompletely()
-      
-      // CRITICAL FIX: Wait for all I/O operations to complete before clearing writeBuffer
-      await this._waitForIOCompletion()
-      
-      // CRITICAL FIX: Verify write buffer is empty after I/O completion
-      // But allow for ongoing insertions during high-volume scenarios
-      if (this.writeBuffer.length > 0) {
-        if (this.opts.debugMode) {
-          console.log(`💾 Save: WriteBuffer still has ${this.writeBuffer.length} items after flush - this may indicate ongoing insertions`)
-        }
-        
-        // If we have a reasonable number of items, continue processing
-        if (this.writeBuffer.length < 10000) { // Reasonable threshold
-          if (this.opts.debugMode) {
-            console.log(`💾 Save: Continuing to process remaining ${this.writeBuffer.length} items`)
-          }
-          // Continue with the save process - the remaining items will be included in the final save
-        } else {
-          // Too many items remaining - likely a real problem
-          throw new Error(`WriteBuffer has too many items after flush: ${this.writeBuffer.length} items remaining (threshold: 10000)`)
-        }
+      // CRITICAL FIX: DO NOT flush writeBuffer before processing existing records
+      // This prevents duplicating updated records in the file.
+      // The _streamExistingRecords() will handle replacing old records with updated ones from writeBufferSnapshot.
+      // After processing, all records (existing + updated + new) will be written to file in one operation.
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: writeBufferSnapshot captured with ${writeBufferSnapshot.length} records (will be processed with existing records)`)
       }
       
       // OPTIMIZATION: Parallel operations - cleanup and data preparation
       let allData = []
       let orphanedCount = 0
       
-      // Check if there are new records to save (after flush, writeBuffer should be empty)
-      // CRITICAL FIX: Also check writeBufferSnapshot.length > 0 to handle updates/deletes
-      // that were in writeBuffer before flush but are now in snapshot
+      // Check if there are records to save from writeBufferSnapshot
+      // CRITICAL FIX: Process writeBufferSnapshot records (both new and updated) with existing records
+      // Updated records will replace old ones via _streamExistingRecords, new records will be added
       if (this.opts.debugMode) {
         console.log(`💾 Save: writeBuffer.length=${this.writeBuffer.length}, writeBufferSnapshot.length=${writeBufferSnapshot.length}`)
       }
@@ -1397,11 +1389,49 @@ class Database extends EventEmitter {
               // CRITICAL FIX: Normalize IDs to strings for consistent comparison
               const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)))
               
+              // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+              const existingRecordsById = new Map()
+              existingRecords.forEach(r => {
+                if (r && r.id) {
+                  existingRecordsById.set(String(r.id), r)
+                }
+              })
+              
               // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+              // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
               for (const record of writeBufferSnapshot) {
-                if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(String(record.id))) {
+                if (!record || !record.id) continue
+                if (deletedIdsSnapshot.has(String(record.id))) continue
+                
+                const recordIdStr = String(record.id)
+                const existingRecord = existingRecordsById.get(recordIdStr)
+                
+                if (!existingRecord) {
                   // This is a new record, not an update
                   allData.push(record)
+                  if (this.opts.debugMode) {
+                    console.log(`💾 Save: Adding NEW record to allData:`, { id: recordIdStr, price: record.price, app_id: record.app_id, currency: record.currency })
+                  }
+                } else {
+                  // This is an update - verify that existingRecords contains the updated version
+                  // If not, replace it (this handles edge cases where substitution might have failed)
+                  const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr)
+                  if (existingIndex !== -1) {
+                    // Verify if the existing record is actually the updated one
+                    // Compare key fields to detect if replacement is needed
+                    const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record)
+                    if (needsReplacement) {
+                      if (this.opts.debugMode) {
+                        console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                          old: { id: String(allData[existingIndex].id), price: allData[existingIndex].price },
+                          new: { id: recordIdStr, price: record.price }
+                        })
+                      }
+                      allData[existingIndex] = record
+                    } else if (this.opts.debugMode) {
+                      console.log(`💾 Save: Record already correctly updated in allData:`, { id: recordIdStr })
+                    }
+                  }
                 }
               }
             })
@@ -1452,13 +1482,52 @@ class Database extends EventEmitter {
                 allData = [...existingRecords]
                 
                 // OPTIMIZATION: Use Set for faster lookups of existing record IDs
-                const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => r.id))
+                // CRITICAL FIX: Normalize IDs to strings for consistent comparison
+                const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)))
+                
+                // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+                const existingRecordsById = new Map()
+                existingRecords.forEach(r => {
+                  if (r && r.id) {
+                    existingRecordsById.set(String(r.id), r)
+                  }
+                })
                 
                 // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+                // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
                 for (const record of writeBufferSnapshot) {
-                  if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(record.id)) {
+                  if (!record || !record.id) continue
+                  if (deletedIdsSnapshot.has(String(record.id))) continue
+                  
+                  const recordIdStr = String(record.id)
+                  const existingRecord = existingRecordsById.get(recordIdStr)
+                  
+                  if (!existingRecord) {
                     // This is a new record, not an update
                     allData.push(record)
+                    if (this.opts.debugMode) {
+                      console.log(`💾 Save: Adding NEW record to allData:`, { id: recordIdStr, price: record.price, app_id: record.app_id, currency: record.currency })
+                    }
+                  } else {
+                    // This is an update - verify that existingRecords contains the updated version
+                    // If not, replace it (this handles edge cases where substitution might have failed)
+                    const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr)
+                    if (existingIndex !== -1) {
+                      // Verify if the existing record is actually the updated one
+                      // Compare key fields to detect if replacement is needed
+                      const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record)
+                      if (needsReplacement) {
+                        if (this.opts.debugMode) {
+                          console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                            old: { id: String(allData[existingIndex].id), price: allData[existingIndex].price },
+                            new: { id: recordIdStr, price: record.price }
+                          })
+                        }
+                        allData[existingIndex] = record
+                      } else if (this.opts.debugMode) {
+                        console.log(`💾 Save: Record already correctly updated in allData:`, { id: recordIdStr })
+                      }
+                    }
                   }
                 }
                 
@@ -1466,10 +1535,11 @@ class Database extends EventEmitter {
                   const updatedCount = writeBufferSnapshot.filter(r => r && r.id && existingRecordIds.has(String(r.id))).length
                   const newCount = writeBufferSnapshot.filter(r => r && r.id && !existingRecordIds.has(String(r.id))).length
                   console.log(`💾 Save: Combined data - existingRecords: ${existingRecords.length}, updatedFromBuffer: ${updatedCount}, newFromBuffer: ${newCount}, total: ${allData.length}`)
-                  console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? r.id : 'no-id'))
+                  console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? String(r.id) : 'no-id'))
                   console.log(`💾 Save: Existing record IDs:`, Array.from(existingRecordIds))
-                  console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? { id: existingRecords[0].id, name: existingRecords[0].name, tags: existingRecords[0].tags } : 'null')
-                  console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? { id: writeBufferSnapshot[0].id, name: writeBufferSnapshot[0].name, tags: writeBufferSnapshot[0].tags } : 'null')
+                  console.log(`💾 Save: All records in allData:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
+                  console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? { id: String(existingRecords[0].id), price: existingRecords[0].price, app_id: existingRecords[0].app_id, currency: existingRecords[0].currency } : 'null')
+                  console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? { id: String(writeBufferSnapshot[0].id), price: writeBufferSnapshot[0].price, app_id: writeBufferSnapshot[0].app_id, currency: writeBufferSnapshot[0].currency } : 'null')
                 }
               }).catch(error => {
                 if (this.opts.debugMode) {
@@ -1545,6 +1615,8 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`)
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`)
+        console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null')
       }
       
@@ -1556,11 +1628,14 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: Serialized ${lines.length} lines`)
+        console.log(`💾 Save: All records in allData after serialization check:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         if (lines.length > 0) {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200))
         }
       }
       
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = []
       let currentOffset = 0
       for (let i = 0; i < lines.length; i++) {
@@ -1571,6 +1646,10 @@ class Database extends EventEmitter {
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8')
       }
       
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`)
+      }
+      
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset
       
@@ -1578,56 +1657,9 @@ class Database extends EventEmitter {
         console.log(`💾 Save: Calculated indexOffset: ${this.indexOffset}, allData.length: ${allData.length}`)
       }
       
-      // OPTIMIZATION: Parallel operations - file writing and index data preparation
-      const parallelWriteOperations = []
-      
-      // Add main file write operation
-      parallelWriteOperations.push(
-        this.fileHandler.writeBatch([jsonlData])
-      )
-      
-      // Add index file operations - ALWAYS save offsets, even without indexed fields
-      if (this.indexManager) {
-        const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
-        
-        // OPTIMIZATION: Parallel data preparation
-        const indexDataPromise = Promise.resolve({
-          index: this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {},
-          offsets: this.offsets, // Save actual offsets for efficient file operations
-          indexOffset: this.indexOffset // Save file size for proper range calculations
-        })
-        
-        // Add term mapping data if needed
-        const termMappingFields = this.getTermMappingFields()
-        if (termMappingFields.length > 0 && this.termManager) {
-          const termDataPromise = this.termManager.saveTerms()
-          
-          // Combine index data and term data
-          const combinedDataPromise = Promise.all([indexDataPromise, termDataPromise]).then(([indexData, termData]) => {
-            indexData.termMapping = termData
-            return indexData
-          })
-          
-          // Add index file write operation
-          parallelWriteOperations.push(
-            combinedDataPromise.then(indexData => {
-              const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
-              return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2))
-            })
-          )
-        } else {
-          // Add index file write operation without term mapping
-          parallelWriteOperations.push(
-            indexDataPromise.then(indexData => {
-              const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
-              return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2))
-            })
-          )
-        }
-      }
-      
-      // Execute parallel write operations
-      await Promise.all(parallelWriteOperations)
+      // CRITICAL FIX: Write main data file first
+      // Index will be saved AFTER reconstruction to ensure it contains correct data
+      await this.fileHandler.writeBatch([jsonlData])
       
       if (this.opts.debugMode) {
         console.log(`💾 Saved ${allData.length} records to ${this.normalizedFile}`)
@@ -1639,11 +1671,15 @@ class Database extends EventEmitter {
       this.shouldSave = false
       this.lastSaveTime = Date.now()
       
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
-        // Rebuild index when records were deleted or updated to maintain consistency
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
         const hadDeletedRecords = deletedIdsSnapshot.size > 0
         const hadUpdatedRecords = writeBufferSnapshot.length > 0
+      
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
+        // Rebuild index when records were deleted or updated to maintain consistency
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
           if (hadDeletedRecords || hadUpdatedRecords) {
             // Clear the index and rebuild it from the saved records
@@ -1661,9 +1697,16 @@ class Database extends EventEmitter {
             
             // Rebuild index from the saved records
             // CRITICAL: Process term mapping for records loaded from file to ensure ${field}Ids are available
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Rebuilding index from ${allData.length} records in allData`)
+            }
             for (let i = 0; i < allData.length; i++) {
               let record = allData[i]
               
+              if (this.opts.debugMode && i < 3) {
+                console.log(`💾 Save: Rebuilding index record[${i}]:`, { id: String(record.id), price: record.price, app_id: record.app_id, currency: record.currency })
+              }
+              
               // CRITICAL FIX: Ensure records have ${field}Ids for term mapping fields
               // Records from writeBuffer already have ${field}Ids from processTermMapping
               // Records from file need to be processed to restore ${field}Ids
@@ -1690,6 +1733,21 @@ class Database extends EventEmitter {
               
               await this.indexManager.add(record, i)
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length
+            if (indexedRecordCount !== this.offsets.length) {
+              console.warn(`⚠️ Index inconsistency detected: indexed ${indexedRecordCount} records but offsets has ${this.offsets.length} entries`)
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length)
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length)
+            }
+
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`)
+            }
           }
         }
         
@@ -1712,6 +1770,22 @@ class Database extends EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId)
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear()
+          this.indexManager.setTotalLines(0)
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`)
+          }
+        }
+        
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId)
+        }
         
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -2231,6 +2305,43 @@ class Database extends EventEmitter {
     }
     
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // Check if index and offsets are synchronized
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0
+        const offsetsLength = this.offsets.length
+
+        if (indexTotalLines !== offsetsLength) {
+          console.warn(`⚠️ Data integrity issue detected: index.totalLines=${indexTotalLines}, offsets.length=${offsetsLength}`)
+          // Auto-correct by updating index totalLines to match offsets
+          this.indexManager.setTotalLines(offsetsLength)
+          if (this.opts.debugMode) {
+            console.log(`🔧 Auto-corrected index totalLines to ${offsetsLength}`)
+          }
+
+          // CRITICAL FIX: Also save the corrected index to prevent persistence of inconsistency
+          // This ensures the .idx.jdb file contains the correct totalLines value
+          try {
+            await this._saveIndexDataToFile()
+            if (this.opts.debugMode) {
+              console.log(`💾 Saved corrected index data to prevent future inconsistencies`)
+            }
+          } catch (error) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Failed to save corrected index: ${error.message}`)
+            }
+          }
+
+          // Verify the fix worked
+          const newIndexTotalLines = this.indexManager.totalLines || 0
+          if (newIndexTotalLines === offsetsLength) {
+            console.log(`✅ Data integrity successfully corrected: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`)
+          } else {
+            console.error(`❌ Data integrity correction failed: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`)
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options)
@@ -2253,36 +2364,25 @@ class Database extends EventEmitter {
       
       
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([
-          Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))),
-          Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))
-        ])
+
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map()
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record)
+        }
+      })
         
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(
-          fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id))
-        )
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record =>
+        record && record.id && !writeBufferMap.has(record.id)
+      )
+
+      // Combine results: file results (filtered) + all writeBuffer results
         allResults = [...filteredFileResults, ...writeBufferResultsWithTerms]
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms]
-        
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id)
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record)
-          }
-        }
-      }
       
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id))
@@ -2540,19 +2640,6 @@ class Database extends EventEmitter {
         
         // CRITICAL FIX: Validate state before update operation
         this.validateState()
-        
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`)
-          }
-          const saveStart = Date.now()
-          await this.save(false) // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`)
-          }
-        }
       
       if (this.opts.debugMode) {
         console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`)
@@ -2565,7 +2652,12 @@ class Database extends EventEmitter {
       }
       
       const updatedRecords = []
-      
+
+      if (this.opts.debugMode) {
+        console.log(`🔄 UPDATE: About to process ${records.length} records`)
+        console.log(`🔄 UPDATE: Records:`, records.map(r => ({ id: r.id, value: r.value })))
+      }
+
       for (const record of records) {
         const recordStart = Date.now()
         if (this.opts.debugMode) {
@@ -2574,11 +2666,20 @@ class Database extends EventEmitter {
         
         const updated = { ...record, ...updateData }
         
+        // DEBUG: Log the update operation details
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: Original record ID: ${record.id}, type: ${typeof record.id}`)
+          console.log(`🔄 UPDATE: Updated record ID: ${updated.id}, type: ${typeof updated.id}`)
+          console.log(`🔄 UPDATE: Update data keys:`, Object.keys(updateData))
+          console.log(`🔄 UPDATE: Updated record keys:`, Object.keys(updated))
+        }
+        
         // Process term mapping for update
         const termMappingStart = Date.now()
         this.processTermMapping(updated, true, record)
         if (this.opts.debugMode) {
           console.log(`🔄 UPDATE: Term mapping completed in ${Date.now() - termMappingStart}ms`)
+          console.log(`🔄 UPDATE: After term mapping - ID: ${updated.id}, type: ${typeof updated.id}`)
         }
         
         // CRITICAL FIX: Remove old terms from index before adding new ones
@@ -2593,13 +2694,20 @@ class Database extends EventEmitter {
         // For records in the file, we need to ensure they are properly marked for replacement
         const index = this.writeBuffer.findIndex(r => r.id === record.id)
         let lineNumber = null
-        
+
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`)
+          console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`)
+          console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id))
+        }
+
         if (index !== -1) {
           // Record is already in writeBuffer, update it
           this.writeBuffer[index] = updated
           lineNumber = this._getAbsoluteLineNumber(index)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         } else {
           // Record is in file, add updated version to writeBuffer
@@ -2609,6 +2717,7 @@ class Database extends EventEmitter {
           lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         }
         
@@ -2644,13 +2753,32 @@ class Database extends EventEmitter {
    */
   async delete(criteria) {
     this._validateInitialization('delete')
-    
+
     return this.operationQueue.enqueue(async () => {
       this.isInsideOperationQueue = true
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState()
-      
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, { operation: 'delete' })
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {})
+          const queryFields = this._extractQueryFields(criteria)
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field))
+
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`)
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`)
+            }
+          }
+        }
+
       const records = await this.find(criteria)
       const deletedIds = []
       
@@ -4084,14 +4212,23 @@ class Database extends EventEmitter {
     // Create a map of updated records for quick lookup
     // CRITICAL FIX: Normalize IDs to strings for consistent comparison
     const updatedRecordsMap = new Map()
-    writeBufferSnapshot.forEach(record => {
+    writeBufferSnapshot.forEach((record, index) => {
       if (record && record.id !== undefined && record.id !== null) {
         // Normalize ID to string for consistent comparison
         const normalizedId = String(record.id)
         updatedRecordsMap.set(normalizedId, record)
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Added to updatedRecordsMap: ID=${normalizedId} (original: ${record.id}, type: ${typeof record.id}), index=${index}`)
+        }
+      } else if (this.opts.debugMode) {
+        console.log(`⚠️ Save: Skipped record in writeBufferSnapshot[${index}] - missing or invalid ID:`, record ? { id: record.id, keys: Object.keys(record) } : 'null')
       }
     })
     
+    if (this.opts.debugMode) {
+      console.log(`💾 Save: updatedRecordsMap size: ${updatedRecordsMap.size}, keys:`, Array.from(updatedRecordsMap.keys()))
+    }
+    
     // OPTIMIZATION: Cache file stats to avoid repeated stat() calls
     let fileSize = 0
     if (this._cachedFileStats && this._cachedFileStats.timestamp > Date.now() - 1000) {
@@ -4245,14 +4382,30 @@ class Database extends EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine)
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema()
+              const idIndex = schema.indexOf('id')
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex]
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length]
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0]
+              }
+            } else {
+              // No schema available, try common positions
             if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
+                // Try position 2 (common in older formats)
               recordId = arrayData[2]
             } else {
-              // For arrays without ID field, use first element as fallback
+                // Fallback: use first element
               recordId = arrayData[0]
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId)
@@ -4301,11 +4454,20 @@ class Database extends EventEmitter {
         
         // CRITICAL FIX: Normalize ID to string for consistent comparison
         const normalizedId = String(recordWithIds.id)
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Checking record ID=${normalizedId} (original: ${recordWithIds.id}, type: ${typeof recordWithIds.id}) in updatedRecordsMap`)
+          console.log(`💾 Save: updatedRecordsMap.has(${normalizedId}): ${updatedRecordsMap.has(normalizedId)}`)
+          if (!updatedRecordsMap.has(normalizedId)) {
+            console.log(`💾 Save: Record ${normalizedId} NOT found in updatedRecordsMap. Available keys:`, Array.from(updatedRecordsMap.keys()))
+          }
+        }
         if (updatedRecordsMap.has(normalizedId)) {
           // Replace with updated version
           const updatedRecord = updatedRecordsMap.get(normalizedId)
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Updated record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`)
+            console.log(`💾 Save: ✅ REPLACING record ${recordWithIds.id} with updated version`)
+            console.log(`💾 Save: Old record:`, { id: recordWithIds.id, price: recordWithIds.price, app_id: recordWithIds.app_id, currency: recordWithIds.currency })
+            console.log(`💾 Save: New record:`, { id: updatedRecord.id, price: updatedRecord.price, app_id: updatedRecord.app_id, currency: updatedRecord.currency })
           }
           return { 
             type: 'updated', 
@@ -4316,7 +4478,7 @@ class Database extends EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`)
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`)
           }
           return { 
             type: 'kept', 
@@ -4327,7 +4489,9 @@ class Database extends EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`)
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`)
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot))
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`)
           }
           return { 
             type: 'deleted', 
@@ -4373,6 +4537,54 @@ class Database extends EventEmitter {
       
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line)
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema()
+                  const idIndex = schema.indexOf('id')
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex])
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line)
+              unchangedRecordId = obj.id ? String(obj.id) : null
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`)
+            }
+            continue
+          }
+          
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`)
+            }
+            deletedOffsets.add(offset)
+            break
+          }
+          
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line)
           keptRecords.push({ offset, type: 'unchanged', line: result.line })