jexidb 2.1.2 → 2.1.4

package/dist/Database.cjs

@@ -1241,11 +1241,11 @@ class IndexManager {
         // This will be handled by the QueryManager's streaming strategy
         continue;
       }
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
+      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
         const fieldIndex = data[field];
 
         // Handle $in operator for array queries
-        if (criteriaValue.$in !== undefined) {
+        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
           const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
 
           // PERFORMANCE: Cache term mapping field check once
@@ -2137,6 +2137,14 @@ class IndexManager {
       // Keep the current index with initialized fields
       return;
     }
+
+    // Restore totalLines from saved data
+    if (index.totalLines !== undefined) {
+      this.totalLines = index.totalLines;
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`);
+      }
+    }
     this.index = processedIndex;
   }
 
@@ -2177,7 +2185,8 @@ class IndexManager {
    */
   toJSON() {
     const serializable = {
-      data: {}
+      data: {},
+      totalLines: this.totalLines
     };
 
     // Check if this is a term mapping field for conversion
@@ -2257,9 +2266,9 @@ class IndexManager {
   }
 }
 
-/**
- * SchemaManager - Manages field schemas for optimized array-based serialization
- * This replaces the need for repeating field names in JSON objects
+/**
+ * SchemaManager - Manages field schemas for optimized array-based serialization
+ * This replaces the need for repeating field names in JSON objects
  */
 class SchemaManager {
   constructor(opts = {}) {
@@ -2277,8 +2286,8 @@ class SchemaManager {
     this.isInitialized = false;
   }
 
-  /**
-   * Initialize schema from options or auto-detect from data
+  /**
+   * Initialize schema from options or auto-detect from data
    */
   initializeSchema(schemaOrData, autoDetect = false) {
     if (this.isInitialized && this.opts.strictSchema) {
@@ -2303,8 +2312,8 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Set explicit schema
+  /**
+   * Set explicit schema
    */
   setSchema(fieldNames) {
     this.schema = [...fieldNames]; // Create copy
@@ -2319,8 +2328,8 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Auto-detect schema from sample data
+  /**
+   * Auto-detect schema from sample data
    */
   autoDetectSchema(sampleData) {
     if (Array.isArray(sampleData)) {
@@ -2341,9 +2350,9 @@ class SchemaManager {
     }
   }
 
-  /**
-   * Initialize schema from database options
-   * Note: schema option is no longer supported, use fields instead
+  /**
+   * Initialize schema from database options
+   * Note: schema option is no longer supported, use fields instead
    */
   initializeFromOptions(opts) {
     // Schema option is no longer supported - fields should be used instead
@@ -2351,8 +2360,8 @@ class SchemaManager {
     // Schema initialization is handled by Database.initializeSchema() using fields
   }
 
-  /**
-   * Add new field to schema (for schema evolution)
+  /**
+   * Add new field to schema (for schema evolution)
    */
   addField(fieldName) {
     if (this.fieldToIndex.has(fieldName)) {
@@ -2368,8 +2377,8 @@ class SchemaManager {
     return newIndex;
   }
 
-  /**
-   * Convert object to array using schema with strict field enforcement
+  /**
+   * Convert object to array using schema with strict field enforcement
    */
   objectToArray(obj) {
     if (!this.isInitialized || !this.opts.enableArraySerialization) {
@@ -2395,8 +2404,8 @@ class SchemaManager {
     return result;
   }
 
-  /**
-   * Convert array back to object using schema
+  /**
+   * Convert array back to object using schema
    */
   arrayToObject(arr) {
     if (!this.isInitialized || !this.opts.enableArraySerialization) {
@@ -2406,31 +2415,36 @@ class SchemaManager {
       return arr; // Don't convert non-arrays
     }
     const obj = {};
+    const idIndex = this.schema.indexOf('id');
+
+    // DISABLED: Schema migration detection was causing field mapping corruption
+    // The logic was incorrectly assuming ID was in first position when it's appended at the end
+    // This caused fields to be shifted incorrectly during object-to-array-to-object conversion
+    let arrayOffset = 0;
 
     // Map array values to object properties
     // Only include fields that are in the schema
-    for (let i = 0; i < Math.min(arr.length, this.schema.length); i++) {
+    for (let i = 0; i < Math.min(arr.length - arrayOffset, this.schema.length); i++) {
       const fieldName = this.schema[i];
+      const arrayIndex = i + arrayOffset;
       // Only include non-undefined values to avoid cluttering the object
-      if (arr[i] !== undefined) {
-        obj[fieldName] = arr[i];
+      if (arr[arrayIndex] !== undefined) {
+        obj[fieldName] = arr[arrayIndex];
       }
     }
 
     // CRITICAL FIX: Always preserve 'id' field if it exists in the original object
     // The 'id' field may not be in the schema but must be preserved
-    // Check if 'id' is in the schema first
-    const idIndex = this.schema.indexOf('id');
     if (idIndex !== -1 && arr[idIndex] !== undefined) {
       // 'id' is in schema and has a value
       obj.id = arr[idIndex];
-    } else if (idIndex === -1 && arr.length > this.schema.length) {
+    } else if (!obj.id && arr.length > this.schema.length + arrayOffset) {
       // 'id' is not in schema but array has extra element(s) - check if last element could be ID
       // This handles cases where ID was added after schema initialization
-      for (let i = this.schema.length; i < arr.length; i++) {
+      for (let i = this.schema.length + arrayOffset; i < arr.length; i++) {
         // Try to infer if this is an ID (string that looks like an ID)
         const potentialId = arr[i];
-        if (potentialId !== undefined && potentialId !== null && typeof potentialId === 'string' && potentialId.length > 0) {
+        if (potentialId !== undefined && potentialId !== null && typeof potentialId === 'string' && potentialId.length > 0 && potentialId.length < 100) {
           obj.id = potentialId;
           break; // Use first potential ID found
         }
@@ -2439,43 +2453,43 @@ class SchemaManager {
     return obj;
   }
 
-  /**
-   * Get field index by name
+  /**
+   * Get field index by name
    */
   getFieldIndex(fieldName) {
     return this.fieldToIndex.get(fieldName);
   }
 
-  /**
-   * Get field name by index
+  /**
+   * Get field name by index
    */
   getFieldName(index) {
     return this.indexToField.get(index);
   }
 
-  /**
-   * Check if field exists in schema
+  /**
+   * Check if field exists in schema
    */
   hasField(fieldName) {
     return this.fieldToIndex.has(fieldName);
   }
 
-  /**
-   * Get schema as array of field names
+  /**
+   * Get schema as array of field names
    */
   getSchema() {
     return [...this.schema]; // Return copy
   }
 
-  /**
-   * Get schema size
+  /**
+   * Get schema size
    */
   getSchemaSize() {
     return this.schema.length;
   }
 
-  /**
-   * Validate that object conforms to schema
+  /**
+   * Validate that object conforms to schema
    */
   validateObject(obj) {
     if (!this.isInitialized || !this.opts.strictSchema) {
@@ -2497,8 +2511,8 @@ class SchemaManager {
     return true;
   }
 
-  /**
-   * Get schema metadata for serialization
+  /**
+   * Get schema metadata for serialization
    */
   getSchemaMetadata() {
     return {
@@ -2509,8 +2523,8 @@ class SchemaManager {
     };
   }
 
-  /**
-   * Reset schema
+  /**
+   * Reset schema
    */
   reset() {
     this.schema = [];
@@ -2520,8 +2534,8 @@ class SchemaManager {
     this.schemaVersion++;
   }
 
-  /**
-   * Get performance statistics
+  /**
+   * Get performance statistics
    */
   getStats() {
     return {
@@ -5134,9 +5148,23 @@ class QueryManager {
       }
     }
 
+    // Handle $not operator - include it if it can be processed by IndexManager
+    if (criteria.$not && typeof criteria.$not === 'object') {
+      // Check if $not condition contains only indexable fields
+      const notFields = Object.keys(criteria.$not);
+      const allNotFieldsIndexed = notFields.every(field => this.indexManager.opts.indexes && this.indexManager.opts.indexes[field]);
+      if (allNotFieldsIndexed && notFields.length > 0) {
+        // Extract indexable criteria from $not condition
+        const indexableNotCriteria = this._extractIndexableCriteria(criteria.$not);
+        if (Object.keys(indexableNotCriteria).length > 0) {
+          indexableCriteria.$not = indexableNotCriteria;
+        }
+      }
+    }
+
     // Handle regular field conditions
     for (const [field, condition] of Object.entries(criteria)) {
-      if (field.startsWith('$')) continue; // Skip logical operators
+      if (field.startsWith('$')) continue; // Skip logical operators (already handled above)
 
       // RegExp conditions cannot be pre-filtered using indices
       if (condition instanceof RegExp) {
@@ -5484,47 +5512,82 @@ class QueryManager {
     // Read specific records using the line numbers
     if (lineNumbers.size > 0) {
       const lineNumbersArray = Array.from(lineNumbers);
-      const ranges = this.database.getRanges(lineNumbersArray);
-      const groupedRanges = await this.database.fileHandler.groupedRanges(ranges);
-      const fs = await import('fs');
-      const fd = await fs.promises.open(this.database.fileHandler.file, 'r');
-      try {
-        for (const groupedRange of groupedRanges) {
-          var _iteratorAbruptCompletion2 = false;
-          var _didIteratorError2 = false;
-          var _iteratorError2;
+      const persistedCount = Array.isArray(this.database.offsets) ? this.database.offsets.length : 0;
+
+      // Separate lineNumbers into file records and writeBuffer records
+      const fileLineNumbers = [];
+      const writeBufferLineNumbers = [];
+      for (const lineNumber of lineNumbersArray) {
+        if (lineNumber >= persistedCount) {
+          // This lineNumber points to writeBuffer
+          writeBufferLineNumbers.push(lineNumber);
+        } else {
+          // This lineNumber points to file
+          fileLineNumbers.push(lineNumber);
+        }
+      }
+
+      // Read records from file
+      if (fileLineNumbers.length > 0) {
+        const ranges = this.database.getRanges(fileLineNumbers);
+        if (ranges.length > 0) {
+          const groupedRanges = await this.database.fileHandler.groupedRanges(ranges);
+          const fs = await import('fs');
+          const fd = await fs.promises.open(this.database.fileHandler.file, 'r');
           try {
-            for (var _iterator2 = _asyncIterator(this.database.fileHandler.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = await _iterator2.next()).done; _iteratorAbruptCompletion2 = false) {
-              const row = _step2.value;
-              {
+            for (const groupedRange of groupedRanges) {
+              var _iteratorAbruptCompletion2 = false;
+              var _didIteratorError2 = false;
+              var _iteratorError2;
+              try {
+                for (var _iterator2 = _asyncIterator(this.database.fileHandler.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = await _iterator2.next()).done; _iteratorAbruptCompletion2 = false) {
+                  const row = _step2.value;
+                  {
+                    try {
+                      const record = this.database.serializer.deserialize(row.line);
+                      const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
+                      results.push(recordWithTerms);
+                      if (limit && results.length >= limit) break;
+                    } catch (error) {
+                      // Skip invalid lines
+                    }
+                  }
+                }
+              } catch (err) {
+                _didIteratorError2 = true;
+                _iteratorError2 = err;
+              } finally {
                 try {
-                  const record = this.database.serializer.deserialize(row.line);
-                  const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
-                  results.push(recordWithTerms);
-                  if (limit && results.length >= limit) break;
-                } catch (error) {
-                  // Skip invalid lines
+                  if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
+                    await _iterator2.return();
+                  }
+                } finally {
+                  if (_didIteratorError2) {
+                    throw _iteratorError2;
+                  }
                 }
               }
+              if (limit && results.length >= limit) break;
             }
-          } catch (err) {
-            _didIteratorError2 = true;
-            _iteratorError2 = err;
           } finally {
-            try {
-              if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
-                await _iterator2.return();
-              }
-            } finally {
-              if (_didIteratorError2) {
-                throw _iteratorError2;
-              }
-            }
+            await fd.close();
           }
+        }
+      }
+
+      // Read records from writeBuffer
+      if (writeBufferLineNumbers.length > 0 && this.database.writeBuffer) {
+        for (const lineNumber of writeBufferLineNumbers) {
           if (limit && results.length >= limit) break;
+          const writeBufferIndex = lineNumber - persistedCount;
+          if (writeBufferIndex >= 0 && writeBufferIndex < this.database.writeBuffer.length) {
+            const record = this.database.writeBuffer[writeBufferIndex];
+            if (record) {
+              const recordWithTerms = options.restoreTerms !== false ? this.database.restoreTermIdsAfterDeserialization(record) : record;
+              results.push(recordWithTerms);
+            }
+          }
         }
-      } finally {
-        await fd.close();
       }
     }
     if (options.orderBy) {
@@ -5944,8 +6007,8 @@ class QueryManager {
       }
     }
     const allFieldsIndexed = Object.keys(criteria).every(field => {
-      // Skip $and as it's handled separately above
-      if (field === '$and') return true;
+      // Skip $and and $not as they're handled separately above
+      if (field === '$and' || field === '$not') return true;
       if (!this.opts.indexes || !this.opts.indexes[field]) {
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' not indexed. Available indexes:`, Object.keys(this.opts.indexes || {}));
@@ -5963,7 +6026,7 @@ class QueryManager {
         }
         return false;
       }
-      if (typeof condition === 'object' && !Array.isArray(condition)) {
+      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
         const operators = Object.keys(condition).map(op => normalizeOperator(op));
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' has operators:`, operators);
@@ -6260,7 +6323,7 @@ class QueryManager {
       if (field.startsWith('$')) continue;
       if (termMappingFields.includes(field)) {
         // For term mapping fields, simple equality or $in queries work well
-        if (typeof condition === 'string' || typeof condition === 'object' && condition.$in && Array.isArray(condition.$in)) {
+        if (typeof condition === 'string' || typeof condition === 'object' && condition !== null && condition.$in && Array.isArray(condition.$in)) {
           return true;
         }
       }
@@ -8158,6 +8221,14 @@ class Database extends events.EventEmitter {
 
       // Manual save is now the default behavior
 
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length);
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`);
+        }
+      }
       this.initialized = true;
       this.emit('initialized');
       if (this.opts.debugMode) {
@@ -8654,39 +8725,21 @@ class Database extends events.EventEmitter {
         this.pendingIndexUpdates = [];
       }
 
-      // CRITICAL FIX: Flush write buffer completely after capturing snapshot
-      await this._flushWriteBufferCompletely();
-
-      // CRITICAL FIX: Wait for all I/O operations to complete before clearing writeBuffer
-      await this._waitForIOCompletion();
-
-      // CRITICAL FIX: Verify write buffer is empty after I/O completion
-      // But allow for ongoing insertions during high-volume scenarios
-      if (this.writeBuffer.length > 0) {
-        if (this.opts.debugMode) {
-          console.log(`💾 Save: WriteBuffer still has ${this.writeBuffer.length} items after flush - this may indicate ongoing insertions`);
-        }
-
-        // If we have a reasonable number of items, continue processing
-        if (this.writeBuffer.length < 10000) {
-          // Reasonable threshold
-          if (this.opts.debugMode) {
-            console.log(`💾 Save: Continuing to process remaining ${this.writeBuffer.length} items`);
-          }
-          // Continue with the save process - the remaining items will be included in the final save
-        } else {
-          // Too many items remaining - likely a real problem
-          throw new Error(`WriteBuffer has too many items after flush: ${this.writeBuffer.length} items remaining (threshold: 10000)`);
-        }
+      // CRITICAL FIX: DO NOT flush writeBuffer before processing existing records
+      // This prevents duplicating updated records in the file.
+      // The _streamExistingRecords() will handle replacing old records with updated ones from writeBufferSnapshot.
+      // After processing, all records (existing + updated + new) will be written to file in one operation.
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: writeBufferSnapshot captured with ${writeBufferSnapshot.length} records (will be processed with existing records)`);
       }
 
       // OPTIMIZATION: Parallel operations - cleanup and data preparation
       let allData = [];
       let orphanedCount = 0;
 
-      // Check if there are new records to save (after flush, writeBuffer should be empty)
-      // CRITICAL FIX: Also check writeBufferSnapshot.length > 0 to handle updates/deletes
-      // that were in writeBuffer before flush but are now in snapshot
+      // Check if there are records to save from writeBufferSnapshot
+      // CRITICAL FIX: Process writeBufferSnapshot records (both new and updated) with existing records
+      // Updated records will replace old ones via _streamExistingRecords, new records will be added
       if (this.opts.debugMode) {
         console.log(`💾 Save: writeBuffer.length=${this.writeBuffer.length}, writeBufferSnapshot.length=${writeBufferSnapshot.length}`);
       }
@@ -8729,11 +8782,60 @@ class Database extends events.EventEmitter {
             // CRITICAL FIX: Normalize IDs to strings for consistent comparison
             const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)));
 
+            // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+            const existingRecordsById = new Map();
+            existingRecords.forEach(r => {
+              if (r && r.id) {
+                existingRecordsById.set(String(r.id), r);
+              }
+            });
+
             // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+            // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
             for (const record of writeBufferSnapshot) {
-              if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(String(record.id))) {
+              if (!record || !record.id) continue;
+              if (deletedIdsSnapshot.has(String(record.id))) continue;
+              const recordIdStr = String(record.id);
+              const existingRecord = existingRecordsById.get(recordIdStr);
+              if (!existingRecord) {
                 // This is a new record, not an update
                 allData.push(record);
+                if (this.opts.debugMode) {
+                  console.log(`💾 Save: Adding NEW record to allData:`, {
+                    id: recordIdStr,
+                    price: record.price,
+                    app_id: record.app_id,
+                    currency: record.currency
+                  });
+                }
+              } else {
+                // This is an update - verify that existingRecords contains the updated version
+                // If not, replace it (this handles edge cases where substitution might have failed)
+                const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr);
+                if (existingIndex !== -1) {
+                  // Verify if the existing record is actually the updated one
+                  // Compare key fields to detect if replacement is needed
+                  const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record);
+                  if (needsReplacement) {
+                    if (this.opts.debugMode) {
+                      console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                        old: {
+                          id: String(allData[existingIndex].id),
+                          price: allData[existingIndex].price
+                        },
+                        new: {
+                          id: recordIdStr,
+                          price: record.price
+                        }
+                      });
+                    }
+                    allData[existingIndex] = record;
+                  } else if (this.opts.debugMode) {
+                    console.log(`💾 Save: Record already correctly updated in allData:`, {
+                      id: recordIdStr
+                    });
+                  }
+                }
               }
             }
           }));
@@ -8780,30 +8882,88 @@ class Database extends events.EventEmitter {
               allData = [...existingRecords];
 
               // OPTIMIZATION: Use Set for faster lookups of existing record IDs
-              const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => r.id));
+              // CRITICAL FIX: Normalize IDs to strings for consistent comparison
+              const existingRecordIds = new Set(existingRecords.filter(r => r && r.id).map(r => String(r.id)));
+
+              // CRITICAL FIX: Create a map of records in existingRecords by ID for comparison
+              const existingRecordsById = new Map();
+              existingRecords.forEach(r => {
+                if (r && r.id) {
+                  existingRecordsById.set(String(r.id), r);
+                }
+              });
 
               // Add only NEW records from writeBufferSnapshot (not updates, as those are already in existingRecords)
+              // CRITICAL FIX: Also ensure that if an updated record wasn't properly replaced, we replace it now
               for (const record of writeBufferSnapshot) {
-                if (record && record.id && !deletedIdsSnapshot.has(String(record.id)) && !existingRecordIds.has(record.id)) {
+                if (!record || !record.id) continue;
+                if (deletedIdsSnapshot.has(String(record.id))) continue;
+                const recordIdStr = String(record.id);
+                const existingRecord = existingRecordsById.get(recordIdStr);
+                if (!existingRecord) {
                   // This is a new record, not an update
                   allData.push(record);
+                  if (this.opts.debugMode) {
+                    console.log(`💾 Save: Adding NEW record to allData:`, {
+                      id: recordIdStr,
+                      price: record.price,
+                      app_id: record.app_id,
+                      currency: record.currency
+                    });
+                  }
+                } else {
+                  // This is an update - verify that existingRecords contains the updated version
+                  // If not, replace it (this handles edge cases where substitution might have failed)
+                  const existingIndex = allData.findIndex(r => r && r.id && String(r.id) === recordIdStr);
+                  if (existingIndex !== -1) {
+                    // Verify if the existing record is actually the updated one
+                    // Compare key fields to detect if replacement is needed
+                    const needsReplacement = JSON.stringify(allData[existingIndex]) !== JSON.stringify(record);
+                    if (needsReplacement) {
+                      if (this.opts.debugMode) {
+                        console.log(`💾 Save: REPLACING existing record with updated version in allData:`, {
+                          old: {
+                            id: String(allData[existingIndex].id),
+                            price: allData[existingIndex].price
+                          },
+                          new: {
+                            id: recordIdStr,
+                            price: record.price
+                          }
+                        });
+                      }
+                      allData[existingIndex] = record;
+                    } else if (this.opts.debugMode) {
+                      console.log(`💾 Save: Record already correctly updated in allData:`, {
+                        id: recordIdStr
+                      });
+                    }
+                  }
                 }
               }
               if (this.opts.debugMode) {
                 const updatedCount = writeBufferSnapshot.filter(r => r && r.id && existingRecordIds.has(String(r.id))).length;
                 const newCount = writeBufferSnapshot.filter(r => r && r.id && !existingRecordIds.has(String(r.id))).length;
                 console.log(`💾 Save: Combined data - existingRecords: ${existingRecords.length}, updatedFromBuffer: ${updatedCount}, newFromBuffer: ${newCount}, total: ${allData.length}`);
-                console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? r.id : 'no-id'));
+                console.log(`💾 Save: WriteBuffer record IDs:`, writeBufferSnapshot.map(r => r && r.id ? String(r.id) : 'no-id'));
                 console.log(`💾 Save: Existing record IDs:`, Array.from(existingRecordIds));
+                console.log(`💾 Save: All records in allData:`, allData.map(r => r && r.id ? {
+                  id: String(r.id),
+                  price: r.price,
+                  app_id: r.app_id,
+                  currency: r.currency
+                } : 'no-id'));
                 console.log(`💾 Save: Sample existing record:`, existingRecords[0] ? {
-                  id: existingRecords[0].id,
-                  name: existingRecords[0].name,
-                  tags: existingRecords[0].tags
+                  id: String(existingRecords[0].id),
+                  price: existingRecords[0].price,
+                  app_id: existingRecords[0].app_id,
+                  currency: existingRecords[0].currency
                 } : 'null');
                 console.log(`💾 Save: Sample writeBuffer record:`, writeBufferSnapshot[0] ? {
-                  id: writeBufferSnapshot[0].id,
-                  name: writeBufferSnapshot[0].name,
-                  tags: writeBufferSnapshot[0].tags
+                  id: String(writeBufferSnapshot[0].id),
+                  price: writeBufferSnapshot[0].price,
+                  app_id: writeBufferSnapshot[0].app_id,
+                  currency: writeBufferSnapshot[0].currency
                 } : 'null');
               }
             }).catch(error => {
@@ -8874,6 +9034,13 @@ class Database extends events.EventEmitter {
       });
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`);
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`);
+        console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? {
+          id: String(r.id),
+          price: r.price,
+          app_id: r.app_id,
+          currency: r.currency
+        } : 'no-id'));
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null');
       }
       const jsonlData = cleanedData.length > 0 ? this.serializer.serializeBatch(cleanedData) : '';
@@ -8881,10 +9048,19 @@ class Database extends events.EventEmitter {
       const lines = jsonlString.split('\n').filter(line => line.trim());
       if (this.opts.debugMode) {
         console.log(`💾 Save: Serialized ${lines.length} lines`);
+        console.log(`💾 Save: All records in allData after serialization check:`, allData.map(r => r && r.id ? {
+          id: String(r.id),
+          price: r.price,
+          app_id: r.app_id,
+          currency: r.currency
+        } : 'no-id'));
         if (lines.length > 0) {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200));
         }
       }
+
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = [];
       let currentOffset = 0;
       for (let i = 0; i < lines.length; i++) {
@@ -8894,6 +9070,9 @@ class Database extends events.EventEmitter {
         const lineWithNewline = lines[i] + '\n';
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8');
       }
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`);
+      }
 
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset;
@@ -8901,51 +9080,9 @@ class Database extends events.EventEmitter {
         console.log(`💾 Save: Calculated indexOffset: ${this.indexOffset}, allData.length: ${allData.length}`);
       }
 
-      // OPTIMIZATION: Parallel operations - file writing and index data preparation
-      const parallelWriteOperations = [];
-
-      // Add main file write operation
-      parallelWriteOperations.push(this.fileHandler.writeBatch([jsonlData]));
-
-      // Add index file operations - ALWAYS save offsets, even without indexed fields
-      if (this.indexManager) {
-        const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb');
-
-        // OPTIMIZATION: Parallel data preparation
-        const indexDataPromise = Promise.resolve({
-          index: this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {},
-          offsets: this.offsets,
-          // Save actual offsets for efficient file operations
-          indexOffset: this.indexOffset // Save file size for proper range calculations
-        });
-
-        // Add term mapping data if needed
-        const termMappingFields = this.getTermMappingFields();
-        if (termMappingFields.length > 0 && this.termManager) {
-          const termDataPromise = this.termManager.saveTerms();
-
-          // Combine index data and term data
-          const combinedDataPromise = Promise.all([indexDataPromise, termDataPromise]).then(([indexData, termData]) => {
-            indexData.termMapping = termData;
-            return indexData;
-          });
-
-          // Add index file write operation
-          parallelWriteOperations.push(combinedDataPromise.then(indexData => {
-            const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts);
-            return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2));
-          }));
-        } else {
-          // Add index file write operation without term mapping
-          parallelWriteOperations.push(indexDataPromise.then(indexData => {
-            const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts);
-            return idxFileHandler.writeAll(JSON.stringify(indexData, null, 2));
-          }));
-        }
-      }
-
-      // Execute parallel write operations
-      await Promise.all(parallelWriteOperations);
+      // CRITICAL FIX: Write main data file first
+      // Index will be saved AFTER reconstruction to ensure it contains correct data
+      await this.fileHandler.writeBatch([jsonlData]);
       if (this.opts.debugMode) {
         console.log(`💾 Saved ${allData.length} records to ${this.normalizedFile}`);
       }
@@ -8955,22 +9092,45 @@ class Database extends events.EventEmitter {
       this.shouldSave = false;
       this.lastSaveTime = Date.now();
 
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
-        // Rebuild index when records were deleted to maintain consistency
-        const hadDeletedRecords = deletedIdsSnapshot.size > 0;
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
+      const hadDeletedRecords = deletedIdsSnapshot.size > 0;
+      const hadUpdatedRecords = writeBufferSnapshot.length > 0;
+
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
+        // Rebuild index when records were deleted or updated to maintain consistency
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
-          if (hadDeletedRecords) {
-            // Clear the index and rebuild it from the remaining records
+          if (hadDeletedRecords || hadUpdatedRecords) {
+            // Clear the index and rebuild it from the saved records
+            // This ensures that lineNumbers point to the correct positions in the file
             this.indexManager.clear();
             if (this.opts.debugMode) {
-              console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records`);
+              if (hadDeletedRecords && hadUpdatedRecords) {
+                console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records and updating ${writeBufferSnapshot.length} records`);
+              } else if (hadDeletedRecords) {
+                console.log(`🧹 Rebuilding index after removing ${deletedIdsSnapshot.size} deleted records`);
+              } else {
+                console.log(`🧹 Rebuilding index after updating ${writeBufferSnapshot.length} records`);
+              }
             }
 
             // Rebuild index from the saved records
             // CRITICAL: Process term mapping for records loaded from file to ensure ${field}Ids are available
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Rebuilding index from ${allData.length} records in allData`);
+            }
             for (let i = 0; i < allData.length; i++) {
               let record = allData[i];
+              if (this.opts.debugMode && i < 3) {
+                console.log(`💾 Save: Rebuilding index record[${i}]:`, {
+                  id: String(record.id),
+                  price: record.price,
+                  app_id: record.app_id,
+                  currency: record.currency
+                });
+              }
 
               // CRITICAL FIX: Ensure records have ${field}Ids for term mapping fields
               // Records from writeBuffer already have ${field}Ids from processTermMapping
@@ -8997,6 +9157,20 @@ class Database extends events.EventEmitter {
               }
               await this.indexManager.add(record, i);
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length;
+            if (indexedRecordCount !== this.offsets.length) {
+              console.warn(`⚠️ Index inconsistency detected: indexed ${indexedRecordCount} records but offsets has ${this.offsets.length} entries`);
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length);
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length);
+            }
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`);
+            }
           }
         }
 
@@ -9016,6 +9190,22 @@ class Database extends events.EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId);
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear();
+          this.indexManager.setTotalLines(0);
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`);
+          }
+        }
+
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId);
+        }
 
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -9077,12 +9267,21 @@ class Database extends events.EventEmitter {
               this.termManager.decrementTermCount(termId);
             }
           } else if (oldRecord[field] && Array.isArray(oldRecord[field])) {
-            // Use terms to decrement (fallback for backward compatibility)
-            for (const term of oldRecord[field]) {
-              const termId = this.termManager.termToId.get(term);
-              if (termId) {
+            // Check if field contains term IDs (numbers) or terms (strings)
+            const firstValue = oldRecord[field][0];
+            if (typeof firstValue === 'number') {
+              // Field contains term IDs (from find with restoreTerms: false)
+              for (const termId of oldRecord[field]) {
                 this.termManager.decrementTermCount(termId);
               }
+            } else if (typeof firstValue === 'string') {
+              // Field contains terms (strings) - convert to term IDs
+              for (const term of oldRecord[field]) {
+                const termId = this.termManager.termToId.get(term);
+                if (termId) {
+                  this.termManager.decrementTermCount(termId);
+                }
+              }
             }
           }
         }
@@ -9323,6 +9522,7 @@ class Database extends events.EventEmitter {
         }
 
         // Apply schema enforcement - convert to array format and back to enforce schema
+        // This will discard any fields not in the schema
         const schemaEnforcedRecord = this.applySchemaEnforcement(record);
 
         // Don't store in this.data - only use writeBuffer and index
@@ -9505,6 +9705,42 @@ class Database extends events.EventEmitter {
       console.log(`🔍 FIND START: criteria=${JSON.stringify(criteria)}, writeBuffer=${this.writeBuffer.length}`);
     }
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // Check if index and offsets are synchronized
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0;
+        const offsetsLength = this.offsets.length;
+        if (indexTotalLines !== offsetsLength) {
+          console.warn(`⚠️ Data integrity issue detected: index.totalLines=${indexTotalLines}, offsets.length=${offsetsLength}`);
+          // Auto-correct by updating index totalLines to match offsets
+          this.indexManager.setTotalLines(offsetsLength);
+          if (this.opts.debugMode) {
+            console.log(`🔧 Auto-corrected index totalLines to ${offsetsLength}`);
+          }
+
+          // CRITICAL FIX: Also save the corrected index to prevent persistence of inconsistency
+          // This ensures the .idx.jdb file contains the correct totalLines value
+          try {
+            await this._saveIndexDataToFile();
+            if (this.opts.debugMode) {
+              console.log(`💾 Saved corrected index data to prevent future inconsistencies`);
+            }
+          } catch (error) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Failed to save corrected index: ${error.message}`);
+            }
+          }
+
+          // Verify the fix worked
+          const newIndexTotalLines = this.indexManager.totalLines || 0;
+          if (newIndexTotalLines === offsetsLength) {
+            console.log(`✅ Data integrity successfully corrected: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`);
+          } else {
+            console.error(`❌ Data integrity correction failed: index.totalLines=${newIndexTotalLines}, offsets.length=${offsetsLength}`);
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options);
@@ -9521,31 +9757,23 @@ class Database extends events.EventEmitter {
       const writeBufferResultsWithTerms = options.restoreTerms !== false ? writeBufferResults.map(record => this.restoreTermIdsAfterDeserialization(record)) : writeBufferResults;
 
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults;
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))), Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))]);
 
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id)));
-        allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms];
-
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id);
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record;
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record);
-          }
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map();
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record);
         }
-      }
+      });
+
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record => record && record.id && !writeBufferMap.has(record.id));
+
+      // Combine results: file results (filtered) + all writeBuffer results
+      allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
 
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id));
@@ -9793,19 +10021,6 @@ class Database extends events.EventEmitter {
 
         // CRITICAL FIX: Validate state before update operation
         this.validateState();
-
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`);
-          }
-          const saveStart = Date.now();
-          await this.save(false); // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`);
-          }
-        }
         if (this.opts.debugMode) {
           console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`);
         }
@@ -9818,6 +10033,13 @@ class Database extends events.EventEmitter {
           console.log(`🔄 UPDATE: Find completed in ${Date.now() - findStart}ms, found ${records.length} records`);
         }
         const updatedRecords = [];
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: About to process ${records.length} records`);
+          console.log(`🔄 UPDATE: Records:`, records.map(r => ({
+            id: r.id,
+            value: r.value
+          })));
+        }
         for (const record of records) {
           const recordStart = Date.now();
           if (this.opts.debugMode) {
@@ -9828,19 +10050,12 @@ class Database extends events.EventEmitter {
             ...updateData
           };
 
-          // CRITICAL FIX: Update schema if new fields are added during update
-          // This ensures new fields (like 'status') are included in the serialized format
-          if (this.serializer && this.serializer.schemaManager) {
-            const newFields = Object.keys(updateData).filter(field => !record.hasOwnProperty(field));
-            for (const field of newFields) {
-              if (field !== 'id' && field !== '_') {
-                // Skip special fields
-                this.serializer.schemaManager.addField(field);
-                if (this.opts.debugMode) {
-                  console.log(`🔄 UPDATE: Added new field '${field}' to schema`);
-                }
-              }
-            }
+          // DEBUG: Log the update operation details
+          if (this.opts.debugMode) {
+            console.log(`🔄 UPDATE: Original record ID: ${record.id}, type: ${typeof record.id}`);
+            console.log(`🔄 UPDATE: Updated record ID: ${updated.id}, type: ${typeof updated.id}`);
+            console.log(`🔄 UPDATE: Update data keys:`, Object.keys(updateData));
+            console.log(`🔄 UPDATE: Updated record keys:`, Object.keys(updated));
           }
 
           // Process term mapping for update
@@ -9848,6 +10063,7 @@ class Database extends events.EventEmitter {
           this.processTermMapping(updated, true, record);
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Term mapping completed in ${Date.now() - termMappingStart}ms`);
+            console.log(`🔄 UPDATE: After term mapping - ID: ${updated.id}, type: ${typeof updated.id}`);
           }
 
           // CRITICAL FIX: Remove old terms from index before adding new ones
@@ -9862,12 +10078,18 @@ class Database extends events.EventEmitter {
           // For records in the file, we need to ensure they are properly marked for replacement
           const index = this.writeBuffer.findIndex(r => r.id === record.id);
           let lineNumber = null;
+          if (this.opts.debugMode) {
+            console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`);
+            console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`);
+            console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id));
+          }
           if (index !== -1) {
             // Record is already in writeBuffer, update it
             this.writeBuffer[index] = updated;
             lineNumber = this._getAbsoluteLineNumber(index);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           } else {
             // Record is in file, add updated version to writeBuffer
@@ -9877,6 +10099,7 @@ class Database extends events.EventEmitter {
             lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           }
           const indexUpdateStart = Date.now();
@@ -9912,6 +10135,26 @@ class Database extends events.EventEmitter {
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState();
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, {
+            operation: 'delete'
+          });
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {});
+          const queryFields = this._extractQueryFields(criteria);
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field));
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`);
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`);
+            }
+          }
+        }
         const records = await this.find(criteria);
         const deletedIds = [];
         if (this.opts.debugMode) {
@@ -10821,64 +11064,112 @@ class Database extends events.EventEmitter {
     // Fetch actual records
     const lineNumbers = limitedEntries.map(([lineNumber]) => lineNumber);
     const scoresByLineNumber = new Map(limitedEntries);
-
-    // Use getRanges and fileHandler to read records
-    const ranges = this.getRanges(lineNumbers);
-    const groupedRanges = await this.fileHandler.groupedRanges(ranges);
-    const fs = await import('fs');
-    const fd = await fs.promises.open(this.fileHandler.file, 'r');
+    const persistedCount = Array.isArray(this.offsets) ? this.offsets.length : 0;
+
+    // Separate lineNumbers into file records and writeBuffer records
+    const fileLineNumbers = [];
+    const writeBufferLineNumbers = [];
+    for (const lineNumber of lineNumbers) {
+      if (lineNumber >= persistedCount) {
+        // This lineNumber points to writeBuffer
+        writeBufferLineNumbers.push(lineNumber);
+      } else {
+        // This lineNumber points to file
+        fileLineNumbers.push(lineNumber);
+      }
+    }
     const results = [];
-    try {
-      for (const groupedRange of groupedRanges) {
-        var _iteratorAbruptCompletion3 = false;
-        var _didIteratorError3 = false;
-        var _iteratorError3;
+
+    // Read records from file
+    if (fileLineNumbers.length > 0) {
+      const ranges = this.getRanges(fileLineNumbers);
+      if (ranges.length > 0) {
+        // Create a map from start offset to lineNumber for accurate mapping
+        const startToLineNumber = new Map();
+        for (const range of ranges) {
+          if (range.index !== undefined) {
+            startToLineNumber.set(range.start, range.index);
+          }
+        }
+        const groupedRanges = await this.fileHandler.groupedRanges(ranges);
+        const fs = await import('fs');
+        const fd = await fs.promises.open(this.fileHandler.file, 'r');
         try {
-          for (var _iterator3 = _asyncIterator(this.fileHandler.readGroupedRange(groupedRange, fd)), _step3; _iteratorAbruptCompletion3 = !(_step3 = await _iterator3.next()).done; _iteratorAbruptCompletion3 = false) {
-            const row = _step3.value;
-            {
-              try {
-                const record = this.serializer.deserialize(row.line);
+          for (const groupedRange of groupedRanges) {
+            var _iteratorAbruptCompletion3 = false;
+            var _didIteratorError3 = false;
+            var _iteratorError3;
+            try {
+              for (var _iterator3 = _asyncIterator(this.fileHandler.readGroupedRange(groupedRange, fd)), _step3; _iteratorAbruptCompletion3 = !(_step3 = await _iterator3.next()).done; _iteratorAbruptCompletion3 = false) {
+                const row = _step3.value;
+                {
+                  try {
+                    const record = this.serializer.deserialize(row.line);
 
-                // Get line number from the row
-                const lineNumber = row._ || 0;
+                    // Get line number from the row, fallback to start offset mapping
+                    let lineNumber = row._ !== null && row._ !== undefined ? row._ : startToLineNumber.get(row.start) ?? 0;
 
-                // Restore term IDs to terms
-                const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
+                    // Restore term IDs to terms
+                    const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
 
-                // Add line number
-                recordWithTerms._ = lineNumber;
+                    // Add line number
+                    recordWithTerms._ = lineNumber;
 
-                // Add score if includeScore is true
-                if (opts.includeScore) {
-                  recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
+                    // Add score if includeScore is true (default is true)
+                    if (opts.includeScore !== false) {
+                      recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
+                    }
+                    results.push(recordWithTerms);
+                  } catch (error) {
+                    // Skip invalid lines
+                    if (this.opts.debugMode) {
+                      console.error('Error deserializing record in score():', error);
+                    }
+                  }
                 }
-                results.push(recordWithTerms);
-              } catch (error) {
-                // Skip invalid lines
-                if (this.opts.debugMode) {
-                  console.error('Error deserializing record in score():', error);
+              }
+            } catch (err) {
+              _didIteratorError3 = true;
+              _iteratorError3 = err;
+            } finally {
+              try {
+                if (_iteratorAbruptCompletion3 && _iterator3.return != null) {
+                  await _iterator3.return();
+                }
+              } finally {
+                if (_didIteratorError3) {
+                  throw _iteratorError3;
                 }
               }
             }
           }
-        } catch (err) {
-          _didIteratorError3 = true;
-          _iteratorError3 = err;
         } finally {
-          try {
-            if (_iteratorAbruptCompletion3 && _iterator3.return != null) {
-              await _iterator3.return();
-            }
-          } finally {
-            if (_didIteratorError3) {
-              throw _iteratorError3;
+          await fd.close();
+        }
+      }
+    }
+
+    // Read records from writeBuffer
+    if (writeBufferLineNumbers.length > 0 && this.writeBuffer) {
+      for (const lineNumber of writeBufferLineNumbers) {
+        const writeBufferIndex = lineNumber - persistedCount;
+        if (writeBufferIndex >= 0 && writeBufferIndex < this.writeBuffer.length) {
+          const record = this.writeBuffer[writeBufferIndex];
+          if (record) {
+            // Restore term IDs to terms
+            const recordWithTerms = this.restoreTermIdsAfterDeserialization(record);
+
+            // Add line number
+            recordWithTerms._ = lineNumber;
+
+            // Add score if includeScore is true
+            if (opts.includeScore) {
+              recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0;
             }
+            results.push(recordWithTerms);
           }
         }
       }
-    } finally {
-      await fd.close();
     }
 
     // Re-sort results to maintain score order (since reads might be out of order)
@@ -11251,13 +11542,24 @@ class Database extends events.EventEmitter {
     // Create a map of updated records for quick lookup
     // CRITICAL FIX: Normalize IDs to strings for consistent comparison
     const updatedRecordsMap = new Map();
-    writeBufferSnapshot.forEach(record => {
+    writeBufferSnapshot.forEach((record, index) => {
       if (record && record.id !== undefined && record.id !== null) {
         // Normalize ID to string for consistent comparison
         const normalizedId = String(record.id);
         updatedRecordsMap.set(normalizedId, record);
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Added to updatedRecordsMap: ID=${normalizedId} (original: ${record.id}, type: ${typeof record.id}), index=${index}`);
+        }
+      } else if (this.opts.debugMode) {
+        console.log(`⚠️ Save: Skipped record in writeBufferSnapshot[${index}] - missing or invalid ID:`, record ? {
+          id: record.id,
+          keys: Object.keys(record)
+        } : 'null');
       }
     });
+    if (this.opts.debugMode) {
+      console.log(`💾 Save: updatedRecordsMap size: ${updatedRecordsMap.size}, keys:`, Array.from(updatedRecordsMap.keys()));
+    }
 
     // OPTIMIZATION: Cache file stats to avoid repeated stat() calls
     let fileSize = 0;
@@ -11405,14 +11707,30 @@ class Database extends events.EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine);
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
-            if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
-              recordId = arrayData[2];
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema();
+              const idIndex = schema.indexOf('id');
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex];
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             } else {
-              // For arrays without ID field, use first element as fallback
-              recordId = arrayData[0];
+              // No schema available, try common positions
+              if (arrayData.length > 2) {
+                // Try position 2 (common in older formats)
+                recordId = arrayData[2];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId);
@@ -11460,11 +11778,30 @@ class Database extends events.EventEmitter {
 
         // CRITICAL FIX: Normalize ID to string for consistent comparison
         const normalizedId = String(recordWithIds.id);
+        if (this.opts.debugMode) {
+          console.log(`💾 Save: Checking record ID=${normalizedId} (original: ${recordWithIds.id}, type: ${typeof recordWithIds.id}) in updatedRecordsMap`);
+          console.log(`💾 Save: updatedRecordsMap.has(${normalizedId}): ${updatedRecordsMap.has(normalizedId)}`);
+          if (!updatedRecordsMap.has(normalizedId)) {
+            console.log(`💾 Save: Record ${normalizedId} NOT found in updatedRecordsMap. Available keys:`, Array.from(updatedRecordsMap.keys()));
+          }
+        }
         if (updatedRecordsMap.has(normalizedId)) {
           // Replace with updated version
           const updatedRecord = updatedRecordsMap.get(normalizedId);
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Updated record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`);
+            console.log(`💾 Save: ✅ REPLACING record ${recordWithIds.id} with updated version`);
+            console.log(`💾 Save: Old record:`, {
+              id: recordWithIds.id,
+              price: recordWithIds.price,
+              app_id: recordWithIds.app_id,
+              currency: recordWithIds.currency
+            });
+            console.log(`💾 Save: New record:`, {
+              id: updatedRecord.id,
+              price: updatedRecord.price,
+              app_id: updatedRecord.app_id,
+              currency: updatedRecord.currency
+            });
           }
           return {
             type: 'updated',
@@ -11475,7 +11812,7 @@ class Database extends events.EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`);
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`);
           }
           return {
             type: 'kept',
@@ -11486,7 +11823,9 @@ class Database extends events.EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`);
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`);
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot));
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`);
           }
           return {
             type: 'deleted',
@@ -11529,6 +11868,54 @@ class Database extends events.EventEmitter {
       const offset = parseInt(rangeKey);
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null;
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line);
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema();
+                  const idIndex = schema.indexOf('id');
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex]);
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line);
+              unchangedRecordId = obj.id ? String(obj.id) : null;
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`);
+            }
+            continue;
+          }
+
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`);
+            }
+            deletedOffsets.add(offset);
+            break;
+          }
+
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line);
           keptRecords.push({