jexidb 2.1.3 → 2.1.5

package/dist/Database.cjs

@@ -1,7 +1,5 @@
 'use strict';
 
-Object.defineProperty(exports, '__esModule', { value: true });
-
 var events = require('events');
 var asyncMutex = require('async-mutex');
 var fs = require('fs');
@@ -1241,11 +1239,11 @@ class IndexManager {
         // This will be handled by the QueryManager's streaming strategy
         continue;
       }
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
+      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
         const fieldIndex = data[field];
 
         // Handle $in operator for array queries
-        if (criteriaValue.$in !== undefined) {
+        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
           const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
 
           // PERFORMANCE: Cache term mapping field check once
@@ -2137,6 +2135,14 @@ class IndexManager {
       // Keep the current index with initialized fields
       return;
     }
+
+    // Restore totalLines from saved data
+    if (index.totalLines !== undefined) {
+      this.totalLines = index.totalLines;
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`);
+      }
+    }
     this.index = processedIndex;
   }
 
@@ -2177,7 +2183,8 @@ class IndexManager {
    */
   toJSON() {
     const serializable = {
-      data: {}
+      data: {},
+      totalLines: this.totalLines
     };
 
     // Check if this is a term mapping field for conversion
@@ -2408,35 +2415,10 @@ class SchemaManager {
     const obj = {};
     const idIndex = this.schema.indexOf('id');
 
-    // CRITICAL FIX: Handle schema migration where 'id' was first field in old schema
-    // but is not in current schema. Check if first element looks like an ID.
-    // Only do this if:
-    // 1. 'id' is not in current schema
-    // 2. Array has significantly more elements than current schema (2+ extra elements)
-    //    This suggests the old schema had more fields, and 'id' was likely the first
-    // 3. First element is a very short string (max 20 chars) that looks like a generated ID
-    //    (typically alphanumeric, often starting with letters like 'mit...' or similar patterns)
-    // 4. First field in current schema is not 'id' (to avoid false positives)
-    // 5. First element is not an array (to avoid false positives with array fields)
+    // DISABLED: Schema migration detection was causing field mapping corruption
+    // The logic was incorrectly assuming ID was in first position when it's appended at the end
+    // This caused fields to be shifted incorrectly during object-to-array-to-object conversion
     let arrayOffset = 0;
-    if (idIndex === -1 && arr.length >= this.schema.length + 2 && this.schema.length > 0) {
-      // Only apply if array has at least 2 extra elements (suggests old schema had more fields)
-      const firstElement = arr[0];
-      const firstFieldName = this.schema[0];
-
-      // Only apply shift if:
-      // - First field is not 'id'
-      // - First element is a very short string (max 20 chars) that looks like a generated ID
-      // - First element is not an array (to avoid false positives)
-      // - Array has at least 2 extra elements (strong indicator of schema migration)
-      if (firstFieldName !== 'id' && typeof firstElement === 'string' && !Array.isArray(firstElement) && firstElement.length > 0 && firstElement.length <= 20 &&
-      // Very conservative: max 20 chars (typical ID length)
-      /^[a-zA-Z0-9_-]+$/.test(firstElement)) {
-        // First element is likely the ID from old schema
-        obj.id = firstElement;
-        arrayOffset = 1;
-      }
-    }
 
     // Map array values to object properties
     // Only include fields that are in the schema
@@ -2678,12 +2660,15 @@ class Serializer {
    * Advanced serialization with optimized JSON.stringify and buffer pooling
    */
   serializeAdvanced(data, addLinebreak) {
+    // CRITICAL FIX: Sanitize data to remove problematic characters before serialization
+    const sanitizedData = this.sanitizeDataForJSON(data);
+
     // Validate encoding before serialization
-    this.validateEncodingBeforeSerialization(data);
+    this.validateEncodingBeforeSerialization(sanitizedData);
 
     // Use optimized JSON.stringify without buffer pooling
     // NOTE: Buffer pool removed - using direct Buffer creation for simplicity and reliability
-    const json = this.optimizedStringify(data);
+    const json = this.optimizedStringify(sanitizedData);
 
     // CRITICAL FIX: Normalize encoding before creating buffer
     const normalizedJson = this.normalizeEncoding(json);
@@ -2781,6 +2766,44 @@ class Serializer {
   /**
    * Validate encoding before serialization
    */
+  /**
+   * Sanitize data to remove problematic characters that break JSON parsing
+   * CRITICAL FIX: Prevents "Expected ',' or ']'" and "Unterminated string" errors
+   * by removing control characters that cannot be safely represented in JSON
+   */
+  sanitizeDataForJSON(data) {
+    const sanitizeString = str => {
+      if (typeof str !== 'string') return str;
+      return str
+      // Remove control characters that break JSON parsing (but keep \n, \r, \t as they can be escaped)
+      // Remove: NUL, SOH, STX, ETX, EOT, ENQ, ACK, BEL, VT, FF, SO, SI, DLE, DC1-DC4, NAK, SYN, ETB, CAN, EM, SUB, ESC, FS, GS, RS, US, DEL, C1 controls
+      .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/g, '')
+      // Limit string length to prevent performance issues
+      .substring(0, 10000);
+    };
+    const sanitizeArray = arr => {
+      if (!Array.isArray(arr)) return arr;
+      return arr.map(item => this.sanitizeDataForJSON(item)).filter(item => item !== null && item !== undefined && item !== '');
+    };
+    if (typeof data === 'string') {
+      return sanitizeString(data);
+    }
+    if (Array.isArray(data)) {
+      return sanitizeArray(data);
+    }
+    if (data && typeof data === 'object') {
+      const sanitized = {};
+      for (const [key, value] of Object.entries(data)) {
+        const sanitizedValue = this.sanitizeDataForJSON(value);
+        // Only include non-null, non-undefined values
+        if (sanitizedValue !== null && sanitizedValue !== undefined) {
+          sanitized[key] = sanitizedValue;
+        }
+      }
+      return sanitized;
+    }
+    return data;
+  }
   validateEncodingBeforeSerialization(data) {
     const issues = [];
     const checkString = (str, path = '') => {
@@ -2875,12 +2898,15 @@ class Serializer {
    * Standard serialization (fallback)
    */
   serializeStandard(data, addLinebreak) {
+    // CRITICAL FIX: Sanitize data to remove problematic characters before serialization
+    const sanitizedData = this.sanitizeDataForJSON(data);
+
     // Validate encoding before serialization
-    this.validateEncodingBeforeSerialization(data);
+    this.validateEncodingBeforeSerialization(sanitizedData);
 
     // NOTE: Buffer pool removed - using direct Buffer creation for simplicity and reliability
     // CRITICAL: Normalize encoding for all string fields before stringify
-    const normalizedData = this.deepNormalizeEncoding(data);
+    const normalizedData = this.deepNormalizeEncoding(sanitizedData);
     const json = JSON.stringify(normalizedData);
 
     // CRITICAL FIX: Normalize encoding before creating buffer
@@ -3084,11 +3110,14 @@ class Serializer {
    * Batch serialization for multiple records
    */
   serializeBatch(dataArray, opts = {}) {
+    // CRITICAL FIX: Sanitize data to remove problematic characters before serialization
+    const sanitizedDataArray = dataArray.map(data => this.sanitizeDataForJSON(data));
+
     // Validate encoding before serialization
-    this.validateEncodingBeforeSerialization(dataArray);
+    this.validateEncodingBeforeSerialization(sanitizedDataArray);
 
     // Convert all objects to array format for optimization
-    const convertedData = dataArray.map(data => this.convertToArrayFormat(data));
+    const convertedData = sanitizedDataArray.map(data => this.convertToArrayFormat(data));
 
     // Track conversion statistics
     this.serializationStats.arraySerializations += convertedData.filter((item, index) => Array.isArray(item) && typeof dataArray[index] === 'object' && dataArray[index] !== null).length;
@@ -3888,6 +3917,141 @@ class FileHandler {
     }
     return groupedRanges;
   }
+
+  /**
+   * Ensure a line is complete by reading until newline if JSON appears truncated
+   * @param {string} line - The potentially incomplete line
+   * @param {number} fd - File descriptor
+   * @param {number} currentOffset - Current read offset
+   * @returns {string} Complete line
+   */
+  async ensureCompleteLine(line, fd, currentOffset) {
+    // Fast check: if line already ends with newline, it's likely complete
+    if (line.endsWith('\n')) {
+      return line;
+    }
+
+    // Check if the line contains valid JSON by trying to parse it
+    const trimmedLine = line.trim();
+    if (trimmedLine.length === 0) {
+      return line;
+    }
+
+    // Try to parse as JSON to see if it's complete
+    try {
+      JSON.parse(trimmedLine);
+      // If parsing succeeds, the line is complete (but missing newline)
+      // This is unusual but possible, return as-is
+      return line;
+    } catch (jsonError) {
+      // JSON is incomplete, try to read more until we find a newline
+      const bufferSize = 2048; // Read in 2KB chunks for better performance
+      const additionalBuffer = Buffer.allocUnsafe(bufferSize);
+      let additionalOffset = currentOffset;
+      let additionalContent = line;
+
+      // Try reading up to 20KB more to find the newline (increased for safety)
+      const maxAdditionalRead = 20480;
+      let totalAdditionalRead = 0;
+      while (totalAdditionalRead < maxAdditionalRead) {
+        const {
+          bytesRead
+        } = await fd.read(additionalBuffer, 0, bufferSize, additionalOffset);
+        if (bytesRead === 0) {
+          // EOF reached, check if the accumulated content is now valid JSON
+          const finalTrimmed = additionalContent.trim();
+          try {
+            JSON.parse(finalTrimmed);
+            // If parsing succeeds now, return the content
+            return additionalContent;
+          } catch {
+            // Still invalid, return original line to avoid data loss
+            return line;
+          }
+        }
+        const chunk = additionalBuffer.toString('utf8', 0, bytesRead);
+        additionalContent += chunk;
+        totalAdditionalRead += bytesRead;
+
+        // Check if we found a newline in the entire accumulated content
+        const newlineIndex = additionalContent.indexOf('\n', line.length);
+        if (newlineIndex !== -1) {
+          // Found newline, return content up to and including the newline
+          const completeLine = additionalContent.substring(0, newlineIndex + 1);
+
+          // Validate that the complete line contains valid JSON
+          const trimmedComplete = completeLine.trim();
+          try {
+            JSON.parse(trimmedComplete);
+            return completeLine;
+          } catch {
+            // Even with newline, JSON is invalid - this suggests data corruption
+            // Return original line to trigger normal error handling
+            return line;
+          }
+        }
+        additionalOffset += bytesRead;
+      }
+
+      // If we couldn't find a newline within the limit, return the original line
+      // This prevents infinite reading and excessive memory usage
+      return line;
+    }
+  }
+
+  /**
+   * Split content into complete JSON lines, handling special characters and escaped quotes
+   * CRITICAL FIX: Prevents "Expected ',' or ']'" and "Unterminated string" errors by ensuring
+   * each line is a complete, valid JSON object/array, even when containing special characters
+   * @param {string} content - Raw content containing multiple JSON lines
+   * @returns {string[]} Array of complete JSON lines
+   */
+  splitJsonLines(content) {
+    const lines = [];
+    let currentLine = '';
+    let inString = false;
+    let escapeNext = false;
+    let braceCount = 0;
+    let bracketCount = 0;
+    for (let i = 0; i < content.length; i++) {
+      const char = content[i];
+      i > 0 ? content[i - 1] : null;
+      currentLine += char;
+      if (escapeNext) {
+        escapeNext = false;
+        continue;
+      }
+      if (char === '\\') {
+        escapeNext = true;
+        continue;
+      }
+      if (char === '"' && !escapeNext) {
+        inString = !inString;
+        continue;
+      }
+      if (!inString) {
+        if (char === '{') braceCount++;else if (char === '}') braceCount--;else if (char === '[') bracketCount++;else if (char === ']') bracketCount--;else if (char === '\n' && braceCount === 0 && bracketCount === 0) {
+          // Found complete JSON object/array at newline
+          const trimmedLine = currentLine.trim();
+          if (trimmedLine.length > 0) {
+            lines.push(trimmedLine.replace(/\n$/, '')); // Remove trailing newline
+          }
+          currentLine = '';
+          braceCount = 0;
+          bracketCount = 0;
+          inString = false;
+          escapeNext = false;
+        }
+      }
+    }
+
+    // Add remaining content if it's a complete JSON object/array
+    const trimmedLine = currentLine.trim();
+    if (trimmedLine.length > 0 && braceCount === 0 && bracketCount === 0) {
+      lines.push(trimmedLine);
+    }
+    return lines.filter(line => line.trim().length > 0);
+  }
   readGroupedRange(groupedRange, fd) {
     var _this = this;
     return _wrapAsyncGenerator(function* () {
@@ -3915,9 +4079,16 @@ class FileHandler {
           });
         }
 
-        // CRITICAL FIX: Remove trailing newlines and whitespace for single range too
-        // Optimized: Use trimEnd() which efficiently removes all trailing whitespace (faster than manual checks)
-        lineString = lineString.trimEnd();
+        // CRITICAL FIX: For single ranges, check if JSON appears truncated and try to complete it
+        // Only attempt completion if the line doesn't end with newline (indicating possible truncation)
+        if (!lineString.endsWith('\n')) {
+          const completeLine = yield _awaitAsyncGenerator(_this.ensureCompleteLine(lineString, fd, range.start + actualBuffer.length));
+          if (completeLine !== lineString) {
+            lineString = completeLine.trimEnd();
+          }
+        } else {
+          lineString = lineString.trimEnd();
+        }
         yield {
           line: lineString,
           start: range.start,
@@ -3952,10 +4123,29 @@ class FileHandler {
         });
       }
 
+      // CRITICAL FIX: Validate buffer completeness to prevent UTF-8 corruption
+      // When reading non-adjacent ranges, the buffer may be incomplete (last line cut mid-character)
+      const lastNewlineIndex = content.lastIndexOf('\n');
+      if (lastNewlineIndex === -1 || lastNewlineIndex < content.length - 2) {
+        // Buffer may be incomplete - truncate to last complete line
+        if (_this.opts.debugMode) {
+          console.warn(`⚠️ Incomplete buffer detected at offset ${firstRange.start}, truncating to last complete line`);
+        }
+        if (lastNewlineIndex > 0) {
+          content = content.substring(0, lastNewlineIndex + 1);
+        } else {
+          // No complete lines found - may be a serious issue
+          if (_this.opts.debugMode) {
+            console.warn(`⚠️ No complete lines found in buffer at offset ${firstRange.start}`);
+          }
+        }
+      }
+
       // CRITICAL FIX: Handle ranges more carefully to prevent corruption
       if (groupedRange.length === 2 && groupedRange[0].end === groupedRange[1].start) {
-        // Special case: Adjacent ranges - split by newlines to prevent corruption
-        const lines = content.split('\n').filter(line => line.trim().length > 0);
+        // Special case: Adjacent ranges - split by COMPLETE JSON lines, not just newlines
+        // This prevents corruption when lines contain special characters or unescaped quotes
+        const lines = _this.splitJsonLines(content);
         for (let i = 0; i < Math.min(lines.length, groupedRange.length); i++) {
           const range = groupedRange[i];
           yield {
@@ -3991,6 +4181,7 @@ class FileHandler {
 
           // OPTIMIZATION 4: Direct character check instead of regex/trimEnd
           // Remove trailing newlines and whitespace efficiently
+          // CRITICAL FIX: Prevents incomplete JSON line reading that caused "Expected ',' or ']'" parsing errors
           // trimEnd() is actually optimized in V8, but we can check if there's anything to trim first
           const len = rangeContent.length;
           if (len > 0) {
@@ -4002,8 +4193,24 @@ class FileHandler {
             }
           }
           if (rangeContent.length === 0) continue;
+
+          // CRITICAL FIX: For multiple ranges, we cannot safely expand reading
+          // because offsets are pre-calculated. Instead, validate JSON and let
+          // the deserializer handle incomplete lines (which will trigger recovery)
+          const trimmedContent = rangeContent.trim();
+          let finalContent = rangeContent;
+          if (trimmedContent.length > 0) {
+            try {
+              JSON.parse(trimmedContent);
+              // JSON is valid, use as-is
+            } catch (jsonError) {
+              // JSON appears incomplete - this is expected for truncated ranges
+              // Let the deserializer handle it (will trigger streaming recovery if needed)
+              // We don't try to expand reading here because offsets are pre-calculated
+            }
+          }
           yield {
-            line: rangeContent,
+            line: finalContent,
             start: range.start,
             _: range.index !== undefined ? range.index : range._ || null
           };
@@ -4014,41 +4221,47 @@ class FileHandler {
   walk(ranges) {
     var _this2 = this;
     return _wrapAsyncGenerator(function* () {
-      // Check if file exists before trying to read it
-      if (!(yield _awaitAsyncGenerator(_this2.exists()))) {
-        return; // Return empty generator if file doesn't exist
-      }
-      const fd = yield _awaitAsyncGenerator(fs.promises.open(_this2.file, 'r'));
+      // CRITICAL FIX: Acquire file mutex to prevent race conditions with concurrent writes
+      const release = _this2.fileMutex ? yield _awaitAsyncGenerator(_this2.fileMutex.acquire()) : () => {};
       try {
-        const groupedRanges = yield _awaitAsyncGenerator(_this2.groupedRanges(ranges));
-        for (const groupedRange of groupedRanges) {
-          var _iteratorAbruptCompletion2 = false;
-          var _didIteratorError2 = false;
-          var _iteratorError2;
-          try {
-            for (var _iterator2 = _asyncIterator(_this2.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = yield _awaitAsyncGenerator(_iterator2.next())).done; _iteratorAbruptCompletion2 = false) {
-              const row = _step2.value;
-              {
-                yield row;
-              }
-            }
-          } catch (err) {
-            _didIteratorError2 = true;
-            _iteratorError2 = err;
-          } finally {
+        // Check if file exists before trying to read it
+        if (!(yield _awaitAsyncGenerator(_this2.exists()))) {
+          return; // Return empty generator if file doesn't exist
+        }
+        const fd = yield _awaitAsyncGenerator(fs.promises.open(_this2.file, 'r'));
+        try {
+          const groupedRanges = yield _awaitAsyncGenerator(_this2.groupedRanges(ranges));
+          for (const groupedRange of groupedRanges) {
+            var _iteratorAbruptCompletion2 = false;
+            var _didIteratorError2 = false;
+            var _iteratorError2;
             try {
-              if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
-                yield _awaitAsyncGenerator(_iterator2.return());
+              for (var _iterator2 = _asyncIterator(_this2.readGroupedRange(groupedRange, fd)), _step2; _iteratorAbruptCompletion2 = !(_step2 = yield _awaitAsyncGenerator(_iterator2.next())).done; _iteratorAbruptCompletion2 = false) {
+                const row = _step2.value;
+                {
+                  yield row;
+                }
               }
+            } catch (err) {
+              _didIteratorError2 = true;
+              _iteratorError2 = err;
             } finally {
-              if (_didIteratorError2) {
-                throw _iteratorError2;
+              try {
+                if (_iteratorAbruptCompletion2 && _iterator2.return != null) {
+                  yield _awaitAsyncGenerator(_iterator2.return());
+                }
+              } finally {
+                if (_didIteratorError2) {
+                  throw _iteratorError2;
+                }
               }
             }
           }
+        } finally {
+          yield _awaitAsyncGenerator(fd.close());
         }
       } finally {
-        yield _awaitAsyncGenerator(fd.close());
+        release();
       }
     })();
   }
@@ -4174,7 +4387,9 @@ class FileHandler {
           JSON.parse(lines[i]);
           validLines.push(lines[i]);
         } catch (error) {
-          console.warn(`⚠️ Invalid JSON in temp file at line ${i + 1}, skipping:`, lines[i].substring(0, 100));
+          if (this.opts.debugMode) {
+            console.warn(`⚠️ Invalid JSON in temp file at line ${i + 1}, skipping:`, lines[i].substring(0, 100));
+          }
           hasInvalidJson = true;
         }
       }
@@ -4800,7 +5015,9 @@ class FileHandler {
           content = buffer.toString('utf8');
         } catch (error) {
           // If UTF-8 decoding fails, try to recover by finding valid UTF-8 boundaries
-          console.warn(`UTF-8 decoding failed for file ${this.file}, attempting recovery`);
+          if (this.opts.debugMode) {
+            console.warn(`UTF-8 decoding failed for file ${this.file}, attempting recovery`);
+          }
 
           // Find the last complete UTF-8 character
           let validLength = buffer.length;
@@ -6042,7 +6259,7 @@ class QueryManager {
         }
         return false;
       }
-      if (typeof condition === 'object' && !Array.isArray(condition)) {
+      if (typeof condition === 'object' && !Array.isArray(condition) && condition !== null) {
         const operators = Object.keys(condition).map(op => normalizeOperator(op));
         if (this.opts.debugMode) {
           console.log(`🔍 Field '${field}' has operators:`, operators);
@@ -6339,7 +6556,7 @@ class QueryManager {
       if (field.startsWith('$')) continue;
       if (termMappingFields.includes(field)) {
         // For term mapping fields, simple equality or $in queries work well
-        if (typeof condition === 'string' || typeof condition === 'object' && condition.$in && Array.isArray(condition.$in)) {
+        if (typeof condition === 'string' || typeof condition === 'object' && condition !== null && condition.$in && Array.isArray(condition.$in)) {
           return true;
         }
       }
@@ -7889,6 +8106,23 @@ class Database extends events.EventEmitter {
       loadTime: 0
     };
 
+    // Initialize integrity correction tracking
+    this.integrityCorrections = {
+      indexSync: 0,
+      // index.totalLines vs offsets.length corrections
+      indexInconsistency: 0,
+      // Index record count vs offsets mismatch
+      writeBufferForced: 0,
+      // WriteBuffer not cleared after save
+      indexSaveFailures: 0,
+      // Failed to save index data
+      dataIntegrity: 0,
+      // General data integrity issues
+      utf8Recovery: 0,
+      // UTF-8 decoding failures recovered
+      jsonRecovery: 0 // JSON parsing failures recovered
+    };
+
     // Initialize usage stats for QueryManager
     this.usageStats = {
       totalQueries: 0,
@@ -8073,7 +8307,9 @@ class Database extends events.EventEmitter {
         }
       }
       if (arrayStringFields.length > 0) {
-        console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`);
+        if (this.opts.debugMode) {
+          console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`);
+        }
       }
     }
     if (this.opts.debugMode) {
@@ -8114,13 +8350,17 @@ class Database extends events.EventEmitter {
   }
 
   /**
-   * Get term mapping fields from indexes (auto-detected)
+   * Get term mapping fields from configuration or indexes (auto-detected)
    * @returns {string[]} Array of field names that use term mapping
    */
   getTermMappingFields() {
-    if (!this.opts.indexes) return [];
+    // If termMappingFields is explicitly configured, use it
+    if (this.opts.termMappingFields && Array.isArray(this.opts.termMappingFields)) {
+      return [...this.opts.termMappingFields];
+    }
 
-    // Auto-detect fields that benefit from term mapping
+    // Auto-detect fields that benefit from term mapping from indexes
+    if (!this.opts.indexes) return [];
     const termMappingFields = [];
     for (const [field, type] of Object.entries(this.opts.indexes)) {
       // Fields that should use term mapping (only array fields)
@@ -8235,8 +8475,28 @@ class Database extends events.EventEmitter {
         }
       }
 
+      // CRITICAL INTEGRITY CHECK: Ensure IndexManager is consistent with loaded offsets
+      // This must happen immediately after load() to prevent any subsequent operations from seeing inconsistent state
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const currentTotalLines = this.indexManager.totalLines || 0;
+        if (currentTotalLines !== this.offsets.length) {
+          this.indexManager.setTotalLines(this.offsets.length);
+          if (this.opts.debugMode) {
+            console.log(`🔧 Post-load integrity sync: IndexManager totalLines ${currentTotalLines} → ${this.offsets.length}`);
+          }
+        }
+      }
+
       // Manual save is now the default behavior
 
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length);
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`);
+        }
+      }
       this.initialized = true;
       this.emit('initialized');
       if (this.opts.debugMode) {
@@ -8374,11 +8634,11 @@ class Database extends events.EventEmitter {
               this.offsets = parsedIdxData.offsets;
               // CRITICAL FIX: Update IndexManager totalLines to match offsets length
               // This ensures queries and length property work correctly even if offsets are reset later
-              if (this.indexManager && this.offsets.length > 0) {
+              if (this.indexManager) {
                 this.indexManager.setTotalLines(this.offsets.length);
-              }
-              if (this.opts.debugMode) {
-                console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}`);
+                if (this.opts.debugMode) {
+                  console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}, synced IndexManager totalLines`);
+                }
               }
             }
 
@@ -9042,6 +9302,7 @@ class Database extends events.EventEmitter {
       });
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`);
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`);
         console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? {
           id: String(r.id),
           price: r.price,
@@ -9065,6 +9326,9 @@ class Database extends events.EventEmitter {
           console.log(`💾 Save: First line (first 200 chars):`, lines[0].substring(0, 200));
         }
       }
+
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = [];
       let currentOffset = 0;
       for (let i = 0; i < lines.length; i++) {
@@ -9074,6 +9338,9 @@ class Database extends events.EventEmitter {
         const lineWithNewline = lines[i] + '\n';
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8');
       }
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`);
+      }
 
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset;
@@ -9093,11 +9360,15 @@ class Database extends events.EventEmitter {
       this.shouldSave = false;
       this.lastSaveTime = Date.now();
 
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
+      const hadDeletedRecords = deletedIdsSnapshot.size > 0;
+      const hadUpdatedRecords = writeBufferSnapshot.length > 0;
+
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
         // Rebuild index when records were deleted or updated to maintain consistency
-        const hadDeletedRecords = deletedIdsSnapshot.size > 0;
-        const hadUpdatedRecords = writeBufferSnapshot.length > 0;
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
           if (hadDeletedRecords || hadUpdatedRecords) {
             // Clear the index and rebuild it from the saved records
@@ -9154,8 +9425,24 @@ class Database extends events.EventEmitter {
               }
               await this.indexManager.add(record, i);
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length;
+            if (indexedRecordCount !== this.offsets.length) {
+              this.integrityCorrections.indexInconsistency++;
+              console.log(`🔧 Auto-corrected index consistency: ${indexedRecordCount} indexed → ${this.offsets.length} offsets`);
+              if (this.integrityCorrections.indexInconsistency > 5) {
+                console.warn(`⚠️ Frequent index inconsistencies detected (${this.integrityCorrections.indexInconsistency} times)`);
+              }
+
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length);
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length);
+            }
             if (this.opts.debugMode) {
-              console.log(`💾 Save: Index rebuilt with ${allData.length} records`);
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`);
             }
           }
         }
@@ -9176,6 +9463,22 @@ class Database extends events.EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId);
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear();
+          this.indexManager.setTotalLines(0);
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`);
+          }
+        }
+
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId);
+        }
 
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -9675,6 +9978,36 @@ class Database extends events.EventEmitter {
       console.log(`🔍 FIND START: criteria=${JSON.stringify(criteria)}, writeBuffer=${this.writeBuffer.length}`);
     }
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // This is a safety net for unexpected inconsistencies - should rarely trigger
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0;
+        const offsetsLength = this.offsets.length;
+        if (indexTotalLines !== offsetsLength) {
+          // This should be extremely rare - indicates a real bug if it happens frequently
+          this.integrityCorrections.dataIntegrity++;
+
+          // Only show in debug mode - these corrections indicate real issues
+          if (this.opts.debugMode) {
+            console.log(`🔧 Integrity correction needed: index.totalLines ${indexTotalLines} → ${offsetsLength} (${this.integrityCorrections.dataIntegrity} total)`);
+          }
+
+          // Warn if corrections are becoming frequent (indicates a real problem)
+          if (this.integrityCorrections.dataIntegrity > 5) {
+            console.warn(`⚠️ Frequent integrity corrections (${this.integrityCorrections.dataIntegrity} times) - this indicates a systemic issue`);
+          }
+          this.indexManager.setTotalLines(offsetsLength);
+
+          // Try to persist the fix, but don't fail the operation if it doesn't work
+          try {
+            await this._saveIndexDataToFile();
+          } catch (error) {
+            // Just track the failure - don't throw since this is a safety net
+            this.integrityCorrections.indexSaveFailures++;
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options);
@@ -9691,31 +10024,23 @@ class Database extends events.EventEmitter {
       const writeBufferResultsWithTerms = options.restoreTerms !== false ? writeBufferResults.map(record => this.restoreTermIdsAfterDeserialization(record)) : writeBufferResults;
 
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults;
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))), Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))]);
 
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id)));
-        allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms];
-
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id);
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record;
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record);
-          }
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map();
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record);
         }
-      }
+      });
+
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record => record && record.id && !writeBufferMap.has(record.id));
+
+      // Combine results: file results (filtered) + all writeBuffer results
+      allResults = [...filteredFileResults, ...writeBufferResultsWithTerms];
 
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id));
@@ -9963,19 +10288,6 @@ class Database extends events.EventEmitter {
 
         // CRITICAL FIX: Validate state before update operation
         this.validateState();
-
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`);
-          }
-          const saveStart = Date.now();
-          await this.save(false); // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`);
-          }
-        }
         if (this.opts.debugMode) {
           console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`);
         }
@@ -9988,6 +10300,13 @@ class Database extends events.EventEmitter {
           console.log(`🔄 UPDATE: Find completed in ${Date.now() - findStart}ms, found ${records.length} records`);
         }
         const updatedRecords = [];
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: About to process ${records.length} records`);
+          console.log(`🔄 UPDATE: Records:`, records.map(r => ({
+            id: r.id,
+            value: r.value
+          })));
+        }
         for (const record of records) {
           const recordStart = Date.now();
           if (this.opts.debugMode) {
@@ -10026,12 +10345,18 @@ class Database extends events.EventEmitter {
           // For records in the file, we need to ensure they are properly marked for replacement
           const index = this.writeBuffer.findIndex(r => r.id === record.id);
           let lineNumber = null;
+          if (this.opts.debugMode) {
+            console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`);
+            console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`);
+            console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id));
+          }
           if (index !== -1) {
             // Record is already in writeBuffer, update it
             this.writeBuffer[index] = updated;
             lineNumber = this._getAbsoluteLineNumber(index);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           } else {
             // Record is in file, add updated version to writeBuffer
@@ -10041,6 +10366,7 @@ class Database extends events.EventEmitter {
             lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1);
             if (this.opts.debugMode) {
               console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`);
+              console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`);
             }
           }
           const indexUpdateStart = Date.now();
@@ -10076,6 +10402,26 @@ class Database extends events.EventEmitter {
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState();
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, {
+            operation: 'delete'
+          });
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {});
+          const queryFields = this._extractQueryFields(criteria);
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field));
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`);
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`);
+            }
+          }
+        }
         const records = await this.find(criteria);
         const deletedIds = [];
         if (this.opts.debugMode) {
@@ -11628,14 +11974,30 @@ class Database extends events.EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine);
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
-            if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
-              recordId = arrayData[2];
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema();
+              const idIndex = schema.indexOf('id');
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex];
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             } else {
-              // For arrays without ID field, use first element as fallback
-              recordId = arrayData[0];
+              // No schema available, try common positions
+              if (arrayData.length > 2) {
+                // Try position 2 (common in older formats)
+                recordId = arrayData[2];
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0];
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId);
@@ -11717,7 +12079,7 @@ class Database extends events.EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`);
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`);
           }
           return {
             type: 'kept',
@@ -11728,7 +12090,9 @@ class Database extends events.EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`);
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`);
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot));
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`);
           }
           return {
             type: 'deleted',
@@ -11771,6 +12135,54 @@ class Database extends events.EventEmitter {
       const offset = parseInt(rangeKey);
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null;
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line);
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema();
+                  const idIndex = schema.indexOf('id');
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex]);
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2]);
+                  } else {
+                    unchangedRecordId = String(arrayData[0]);
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line);
+              unchangedRecordId = obj.id ? String(obj.id) : null;
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`);
+            }
+            continue;
+          }
+
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`);
+            }
+            deletedOffsets.add(offset);
+            break;
+          }
+
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line);
           keptRecords.push({
@@ -11967,6 +12379,74 @@ class Database extends events.EventEmitter {
     }
     return this._getWriteBufferBaseLineNumber() + writeBufferIndex;
   }
+
+  /**
+   * Attempts to recover a corrupted line by cleaning invalid characters and fixing common JSON issues
+   * @param {string} line - The corrupted line to recover
+   * @returns {string|null} - The recovered line or null if recovery is not possible
+   */
+  _tryRecoverLine(line) {
+    if (!line || typeof line !== 'string') {
+      return null;
+    }
+    try {
+      // Try parsing as-is first
+      JSON.parse(line);
+      return line; // Line is already valid
+    } catch (e) {
+      // Line is corrupted, attempt recovery
+    }
+    let recovered = line.trim();
+
+    // Remove invalid control characters (except \n, \r, \t)
+    recovered = recovered.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '');
+
+    // Try to close unclosed strings
+    // Count quotes and ensure they're balanced
+    const quoteCount = (recovered.match(/"/g) || []).length;
+    if (quoteCount % 2 !== 0) {
+      // Odd number of quotes - try to close the string
+      const lastQuoteIndex = recovered.lastIndexOf('"');
+      if (lastQuoteIndex > 0) {
+        // Check if we're inside a string (not escaped)
+        const beforeLastQuote = recovered.substring(0, lastQuoteIndex);
+        const escapedQuotes = (beforeLastQuote.match(/\\"/g) || []).length;
+        const unescapedQuotes = (beforeLastQuote.match(/"/g) || []).length - escapedQuotes;
+        if (unescapedQuotes % 2 !== 0) {
+          // We're inside an unclosed string - try to close it
+          recovered = recovered + '"';
+        }
+      }
+    }
+
+    // Try to close unclosed arrays/objects
+    const openBraces = (recovered.match(/\{/g) || []).length;
+    const closeBraces = (recovered.match(/\}/g) || []).length;
+    const openBrackets = (recovered.match(/\[/g) || []).length;
+    const closeBrackets = (recovered.match(/\]/g) || []).length;
+
+    // Remove trailing commas before closing braces/brackets
+    recovered = recovered.replace(/,\s*([}\]])/g, '$1');
+
+    // Try to close arrays
+    if (openBrackets > closeBrackets) {
+      recovered = recovered + ']'.repeat(openBrackets - closeBrackets);
+    }
+
+    // Try to close objects
+    if (openBraces > closeBraces) {
+      recovered = recovered + '}'.repeat(openBraces - closeBraces);
+    }
+
+    // Final validation - try to parse
+    try {
+      JSON.parse(recovered);
+      return recovered;
+    } catch (e) {
+      // Recovery failed
+      return null;
+    }
+  }
   _streamingRecoveryGenerator(_x, _x2) {
     var _this = this;
     return _wrapAsyncGenerator(function* (criteria, options, alreadyYielded = 0, map = null, remainingSkipValue = 0) {
@@ -12166,6 +12646,17 @@ class Database extends events.EventEmitter {
 
       // If no data at all, return empty
       if (_this2.indexOffset === 0 && _this2.writeBuffer.length === 0) return;
+
+      // CRITICAL FIX: Wait for any ongoing save operations to complete
+      // This prevents reading partially written data
+      if (_this2.isSaving) {
+        if (_this2.opts.debugMode) {
+          console.log('🔍 walk(): waiting for save operation to complete');
+        }
+        while (_this2.isSaving) {
+          yield _awaitAsyncGenerator(new Promise(resolve => setTimeout(resolve, 10)));
+        }
+      }
       let count = 0;
       let remainingSkip = options.skip || 0;
       let map;
@@ -12311,10 +12802,49 @@ class Database extends events.EventEmitter {
                       } catch (error) {
                         // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
                         // This helps identify data corruption issues
-                        if (1 || _this2.opts.debugMode) {
+                        if (_this2.opts.debugMode) {
                           console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`);
                           console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`);
                         }
+
+                        // CRITICAL FIX: Attempt to recover corrupted line before giving up
+                        const recoveredLine = _this2._tryRecoverLine(row.line);
+                        if (recoveredLine) {
+                          try {
+                            const record = _this2.serializer.deserialize(recoveredLine);
+                            if (record !== null) {
+                              _this2.integrityCorrections.jsonRecovery++;
+                              console.log(`🔧 Recovered corrupted JSON line (${_this2.integrityCorrections.jsonRecovery} recoveries)`);
+                              if (_this2.integrityCorrections.jsonRecovery > 20) {
+                                console.warn(`⚠️ Frequent JSON recovery detected (${_this2.integrityCorrections.jsonRecovery} times) - may indicate data corruption`);
+                              }
+                              const recordWithTerms = _this2.restoreTermIdsAfterDeserialization(record);
+                              if (remainingSkip > 0) {
+                                remainingSkip--;
+                                continue;
+                              }
+                              count++;
+                              if (options.includeOffsets) {
+                                yield {
+                                  entry: recordWithTerms,
+                                  start: row.start,
+                                  _: row._ || 0
+                                };
+                              } else {
+                                if (_this2.opts.includeLinePosition) {
+                                  recordWithTerms._ = row._ || 0;
+                                }
+                                yield recordWithTerms;
+                              }
+                              continue; // Successfully recovered and yielded
+                            }
+                          } catch (recoveryError) {
+                            // Recovery attempt failed, continue with normal error handling
+                            if (_this2.opts.debugMode) {
+                              console.warn(`⚠️ walk(): Line recovery failed: ${recoveryError.message}`);
+                            }
+                          }
+                        }
                         if (!_this2._offsetRecoveryInProgress) {
                           var _iteratorAbruptCompletion5 = false;
                           var _didIteratorError5 = false;
@@ -12463,10 +12993,52 @@ class Database extends events.EventEmitter {
                 } catch (error) {
                   // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
                   // This helps identify data corruption issues
-                  if (1 || _this2.opts.debugMode) {
+                  if (_this2.opts.debugMode) {
                     console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`);
                     console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`);
                   }
+
+                  // CRITICAL FIX: Attempt to recover corrupted line before giving up
+                  const recoveredLine = _this2._tryRecoverLine(row.line);
+                  if (recoveredLine) {
+                    try {
+                      const entry = yield _awaitAsyncGenerator(_this2.serializer.deserialize(recoveredLine, {
+                        compress: _this2.opts.compress,
+                        v8: _this2.opts.v8
+                      }));
+                      if (entry !== null) {
+                        _this2.integrityCorrections.jsonRecovery++;
+                        console.log(`🔧 Recovered corrupted JSON line (${_this2.integrityCorrections.jsonRecovery} recoveries)`);
+                        if (_this2.integrityCorrections.jsonRecovery > 20) {
+                          console.warn(`⚠️ Frequent JSON recovery detected (${_this2.integrityCorrections.jsonRecovery} times) - may indicate data corruption`);
+                        }
+                        const entryWithTerms = _this2.restoreTermIdsAfterDeserialization(entry);
+                        if (remainingSkip > 0) {
+                          remainingSkip--;
+                          continue;
+                        }
+                        count++;
+                        if (options.includeOffsets) {
+                          yield {
+                            entry: entryWithTerms,
+                            start: row.start,
+                            _: row._ || _this2.offsets.findIndex(n => n === row.start)
+                          };
+                        } else {
+                          if (_this2.opts.includeLinePosition) {
+                            entryWithTerms._ = row._ || _this2.offsets.findIndex(n => n === row.start);
+                          }
+                          yield entryWithTerms;
+                        }
+                        continue; // Successfully recovered and yielded
+                      }
+                    } catch (recoveryError) {
+                      // Recovery attempt failed, continue with normal error handling
+                      if (_this2.opts.debugMode) {
+                        console.warn(`⚠️ walk(): Line recovery failed: ${recoveryError.message}`);
+                      }
+                    }
+                  }
                   if (!_this2._offsetRecoveryInProgress) {
                     var _iteratorAbruptCompletion7 = false;
                     var _didIteratorError7 = false;
@@ -12759,7 +13331,11 @@ class Database extends events.EventEmitter {
         await this.save();
         // Ensure writeBuffer is cleared after save
         if (this.writeBuffer.length > 0) {
-          console.warn('⚠️ WriteBuffer not cleared after save() - forcing clear');
+          this.integrityCorrections.writeBufferForced++;
+          console.log(`🔧 Forced WriteBuffer clear after save (${this.writeBuffer.length} items remaining)`);
+          if (this.integrityCorrections.writeBufferForced > 3) {
+            console.warn(`⚠️ Frequent WriteBuffer clear issues detected (${this.integrityCorrections.writeBufferForced} times)`);
+          }
           this.writeBuffer = [];
           this.writeBufferOffsets = [];
           this.writeBufferSizes = [];
@@ -12880,7 +13456,8 @@ class Database extends events.EventEmitter {
           console.log(`💾 Index data saved to ${idxPath}`);
         }
       } catch (error) {
-        console.warn('Failed to save index data:', error.message);
+        this.integrityCorrections.indexSaveFailures++;
+        console.warn(`⚠️ Index save failure (${this.integrityCorrections.indexSaveFailures} times): ${error.message}`);
         throw error; // Re-throw to let caller handle
       }
     }
@@ -12951,4 +13528,3 @@ class Database extends events.EventEmitter {
 }
 
 exports.Database = Database;
-exports.default = Database;