jexidb 1.0.8 → 2.0.0

package/dist/IndexManager.js

@@ -0,0 +1,353 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+/**
+ * IndexManager - In-memory index management
+ * Supports different data types and query operations
+ */
+class IndexManager {
+  constructor(indexes = {}) {
+    this.indexes = {};
+    this.offsets = [];
+    this.recordCount = 0;
+
+    // Initialize indexes based on configuration
+    for (const [field, type] of Object.entries(indexes)) {
+      this.indexes[field] = {
+        type,
+        values: new Map() // Map<value, Set<offsetIndex>>
+      };
+    }
+  }
+
+  /**
+   * Adds a record to the index
+   */
+  addRecord(record, offsetIndex) {
+    this.offsets[offsetIndex] = record._offset || 0;
+    this.recordCount = Math.max(this.recordCount, offsetIndex + 1);
+
+    // Add to indexes
+    for (const [field, index] of Object.entries(this.indexes)) {
+      const value = this.getNestedValue(record, field);
+      if (value !== undefined) {
+        if (!index.values.has(value)) {
+          index.values.set(value, new Set());
+        }
+        index.values.get(value).add(offsetIndex);
+      }
+    }
+  }
+
+  /**
+   * Removes a record from the index
+   */
+  removeRecord(offsetIndex) {
+    // Remove from indexes
+    for (const [field, index] of Object.entries(this.indexes)) {
+      for (const [value, offsetSet] of index.values.entries()) {
+        offsetSet.delete(offsetIndex);
+        if (offsetSet.size === 0) {
+          index.values.delete(value);
+        }
+      }
+    }
+
+    // Mark as removed in the offsets array
+    this.offsets[offsetIndex] = null;
+  }
+
+  /**
+   * Updates a record in the index
+   */
+  updateRecord(record, offsetIndex, oldRecord = null) {
+    // Remove old values from indexes
+    if (oldRecord) {
+      for (const [field, index] of Object.entries(this.indexes)) {
+        const oldValue = this.getNestedValue(oldRecord, field);
+        if (oldValue !== undefined) {
+          const offsetSet = index.values.get(oldValue);
+          if (offsetSet) {
+            offsetSet.delete(offsetIndex);
+            if (offsetSet.size === 0) {
+              index.values.delete(oldValue);
+            }
+          }
+        }
+      }
+    }
+
+    // Add new values to indexes
+    for (const [field, index] of Object.entries(this.indexes)) {
+      const value = this.getNestedValue(record, field);
+      if (value !== undefined) {
+        if (!index.values.has(value)) {
+          index.values.set(value, new Set());
+        }
+        index.values.get(value).add(offsetIndex);
+      }
+    }
+
+    // Update offset
+    this.offsets[offsetIndex] = record._offset || 0;
+  }
+
+  /**
+   * Searches records based on criteria
+   */
+  findRecords(criteria, options = {}) {
+    const {
+      caseInsensitive = false,
+      matchAny = false
+    } = options;
+    if (!criteria || Object.keys(criteria).length === 0) {
+      // Returns all valid records
+      return Array.from(this.offsets.keys()).filter(i => this.offsets[i] !== null);
+    }
+    let matchingOffsets = null;
+    for (const [field, criteriaValue] of Object.entries(criteria)) {
+      const index = this.indexes[field];
+      if (!index) {
+        // If no index exists for this field, we need to scan all records
+        // For now, return empty result if any field doesn't have an index
+        return [];
+      }
+      const fieldOffsets = this.findFieldMatches(field, criteriaValue, caseInsensitive);
+      if (matchingOffsets === null) {
+        matchingOffsets = fieldOffsets;
+      } else if (matchAny) {
+        // Union (OR)
+        matchingOffsets = new Set([...matchingOffsets, ...fieldOffsets]);
+      } else {
+        // Intersection (AND)
+        matchingOffsets = new Set([...matchingOffsets].filter(x => fieldOffsets.has(x)));
+      }
+      if (!matchAny && matchingOffsets.size === 0) {
+        break; // No intersection, stop search
+      }
+    }
+    return matchingOffsets ? Array.from(matchingOffsets) : [];
+  }
+
+  /**
+   * Searches for matches in a specific field
+   */
+  findFieldMatches(field, criteriaValue, caseInsensitive) {
+    const index = this.indexes[field];
+    if (!index) return new Set();
+    const matches = new Set();
+    if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue)) {
+      // Comparison operators
+      for (const [value, offsetSet] of index.values.entries()) {
+        if (this.matchesOperator(value, criteriaValue, caseInsensitive)) {
+          for (const offset of offsetSet) {
+            matches.add(offset);
+          }
+        }
+      }
+    } else {
+      // Direct comparison
+      const values = Array.isArray(criteriaValue) ? criteriaValue : [criteriaValue];
+      for (const searchValue of values) {
+        const offsetSet = index.values.get(searchValue);
+        if (offsetSet) {
+          for (const offset of offsetSet) {
+            matches.add(offset);
+          }
+        }
+      }
+    }
+    return matches;
+  }
+
+  /**
+   * Checks if a value matches the operators
+   */
+  matchesOperator(value, operators, caseInsensitive) {
+    for (const [operator, operatorValue] of Object.entries(operators)) {
+      switch (operator) {
+        case '>':
+          if (value <= operatorValue) return false;
+          break;
+        case '>=':
+          if (value < operatorValue) return false;
+          break;
+        case '<':
+          if (value >= operatorValue) return false;
+          break;
+        case '<=':
+          if (value > operatorValue) return false;
+          break;
+        case '!=':
+          if (value === operatorValue) return false;
+          break;
+        case 'in':
+          if (!Array.isArray(operatorValue) || !operatorValue.includes(value)) return false;
+          break;
+        case 'nin':
+          if (Array.isArray(operatorValue) && operatorValue.includes(value)) return false;
+          break;
+        case 'regex':
+          const regex = new RegExp(operatorValue, caseInsensitive ? 'i' : '');
+          if (!regex.test(String(value))) return false;
+          break;
+        case 'contains':
+          const searchStr = String(operatorValue);
+          const valueStr = String(value);
+          if (caseInsensitive) {
+            if (!valueStr.toLowerCase().includes(searchStr.toLowerCase())) return false;
+          } else {
+            if (!valueStr.includes(searchStr)) return false;
+          }
+          break;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Gets nested value from an object
+   */
+  getNestedValue(obj, path) {
+    return path.split('.').reduce((current, key) => {
+      return current && current[key] !== undefined ? current[key] : undefined;
+    }, obj);
+  }
+
+  /**
+   * Recalculates all offsets after modifications
+   */
+  recalculateOffsets() {
+    let currentOffset = 0;
+    const newOffsets = [];
+    for (let i = 0; i < this.offsets.length; i++) {
+      if (this.offsets[i] !== null) {
+        newOffsets[i] = currentOffset;
+        currentOffset += this.offsets[i];
+      }
+    }
+    this.offsets = newOffsets;
+  }
+
+  /**
+   * Recalculates offsets after a file rewrite
+   * This method should be called after the file has been rewritten
+   */
+  async recalculateOffsetsFromFile(fileHandler) {
+    const newOffsets = [];
+    let currentOffset = 0;
+    let recordIndex = 0;
+
+    // Read the file line by line and recalculate offsets
+    for await (const line of this.walkFile(fileHandler)) {
+      if (line && !line._deleted) {
+        newOffsets[recordIndex] = currentOffset;
+        currentOffset += fileHandler.getByteLength(fileHandler.serialize(line));
+        recordIndex++;
+      }
+    }
+    this.offsets = newOffsets;
+    this.recordCount = recordIndex;
+  }
+
+  /**
+   * Walks through the file to read all records
+   */
+  async *walkFile(fileHandler) {
+    let offset = 0;
+    while (true) {
+      const line = await fileHandler.readLine(offset);
+      if (line === null) break;
+      try {
+        const record = fileHandler.deserialize(line);
+        yield record;
+        offset += fileHandler.getByteLength(line + '\n');
+      } catch (error) {
+        // Skip corrupted lines
+        offset += fileHandler.getByteLength(line + '\n');
+      }
+    }
+  }
+
+  /**
+   * Gets index statistics
+   */
+  getStats() {
+    const stats = {
+      recordCount: this.recordCount,
+      indexCount: Object.keys(this.indexes).length,
+      indexes: {}
+    };
+    for (const [field, index] of Object.entries(this.indexes)) {
+      stats.indexes[field] = {
+        type: index.type,
+        uniqueValues: index.values.size,
+        totalReferences: Array.from(index.values.values()).reduce((sum, set) => sum + set.size, 0)
+      };
+    }
+    return stats;
+  }
+
+  /**
+   * Clears all indexes
+   */
+  clear() {
+    this.indexes = {};
+    this.offsets = [];
+    this.recordCount = 0;
+  }
+
+  /**
+   * Serializes indexes for persistence
+   */
+  serialize() {
+    const serialized = {
+      indexes: {},
+      offsets: this.offsets,
+      recordCount: this.recordCount
+    };
+    for (const [field, index] of Object.entries(this.indexes)) {
+      serialized.indexes[field] = {
+        type: index.type,
+        values: {}
+      };
+      for (const [value, offsetSet] of index.values.entries()) {
+        serialized.indexes[field].values[value] = Array.from(offsetSet);
+      }
+    }
+    return serialized;
+  }
+
+  /**
+   * Deserializes indexes from persistence
+   */
+  deserialize(data) {
+    if (!data || !data.indexes) return;
+    this.indexes = {};
+    this.offsets = data.offsets || [];
+    this.recordCount = data.recordCount || 0;
+    for (const [field, indexData] of Object.entries(data.indexes)) {
+      const type = indexData.type;
+      const values = new Map();
+      for (const [valueStr, offsetArr] of Object.entries(indexData.values)) {
+        let key;
+        if (type === 'number') {
+          key = Number(valueStr);
+        } else if (type === 'boolean') {
+          key = valueStr === 'true';
+        } else {
+          key = valueStr;
+        }
+        values.set(key, new Set(offsetArr));
+      }
+      this.indexes[field] = {
+        type,
+        values
+      };
+    }
+  }
+}
+var _default = exports.default = IndexManager;

package/dist/IntegrityChecker.js

@@ -0,0 +1,364 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _fs = require("fs");
+/**
+ * IntegrityChecker - JSONL file integrity validation
+ * Checks consistency between data, indexes and offsets
+ */
+class IntegrityChecker {
+  constructor(fileHandler, indexManager) {
+    this.fileHandler = fileHandler;
+    this.indexManager = indexManager;
+  }
+
+  /**
+   * Validates the complete integrity of the database
+   */
+  async validateIntegrity(options = {}) {
+    const {
+      checkData = true,
+      checkIndexes = true,
+      checkOffsets = true,
+      verbose = false
+    } = options;
+    const results = {
+      isValid: true,
+      errors: [],
+      warnings: [],
+      stats: {
+        totalRecords: 0,
+        validRecords: 0,
+        corruptedRecords: 0,
+        missingIndexes: 0,
+        orphanedIndexes: 0
+      }
+    };
+    if (verbose) {
+      console.log('🔍 Starting integrity validation...');
+    }
+
+    // Check if file exists
+    const fileExists = await this.fileHandler.exists();
+    if (!fileExists) {
+      results.errors.push('Data file does not exist');
+      results.isValid = false;
+      return results;
+    }
+
+    // Validate file data
+    if (checkData) {
+      const dataResults = await this.validateDataFile(verbose);
+      results.errors.push(...dataResults.errors);
+      results.warnings.push(...dataResults.warnings);
+      results.stats = {
+        ...results.stats,
+        ...dataResults.stats
+      };
+    }
+
+    // Validate indexes
+    if (checkIndexes) {
+      const indexResults = await this.validateIndexes(verbose);
+      results.errors.push(...indexResults.errors);
+      results.warnings.push(...indexResults.warnings);
+      results.stats = {
+        ...results.stats,
+        ...indexResults.stats
+      };
+    }
+
+    // Validate offsets
+    if (checkOffsets) {
+      const offsetResults = await this.validateOffsets(verbose);
+      results.errors.push(...offsetResults.errors);
+      results.warnings.push(...offsetResults.warnings);
+    }
+
+    // Determine if valid
+    results.isValid = results.errors.length === 0;
+    if (verbose) {
+      console.log(`✅ Validation completed: ${results.isValid ? 'VALID' : 'INVALID'}`);
+      console.log(`📊 Statistics:`, results.stats);
+      if (results.errors.length > 0) {
+        console.log(`❌ Errors found:`, results.errors);
+      }
+      if (results.warnings.length > 0) {
+        console.log(`⚠️ Warnings:`, results.warnings);
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Validates the JSONL data file
+   */
+  async validateDataFile(verbose = false) {
+    const results = {
+      errors: [],
+      warnings: [],
+      stats: {
+        totalRecords: 0,
+        validRecords: 0,
+        corruptedRecords: 0
+      }
+    };
+    try {
+      const fd = await _fs.promises.open(this.fileHandler.filePath, 'r');
+      let lineNumber = 0;
+      let offset = 0;
+      const buffer = Buffer.alloc(8192);
+      let lineBuffer = '';
+      try {
+        while (true) {
+          const {
+            bytesRead
+          } = await fd.read(buffer, 0, buffer.length, offset);
+          if (bytesRead === 0) break;
+          const chunk = buffer.toString('utf8', 0, bytesRead);
+          lineBuffer += chunk;
+
+          // Process complete lines
+          let newlineIndex;
+          while ((newlineIndex = lineBuffer.indexOf('\n')) !== -1) {
+            const line = lineBuffer.substring(0, newlineIndex);
+            lineBuffer = lineBuffer.substring(newlineIndex + 1);
+            results.stats.totalRecords++;
+            if (line.trim() === '') {
+              results.warnings.push(`Line ${lineNumber + 1}: Empty line`);
+            } else {
+              try {
+                const record = JSON.parse(line);
+
+                // Check if it's a deleted record
+                if (record._deleted) {
+                  if (verbose) {
+                    console.log(`🗑️ Line ${lineNumber + 1}: Deleted record`);
+                  }
+                } else {
+                  results.stats.validRecords++;
+                  if (verbose) {
+                    console.log(`✅ Line ${lineNumber + 1}: Valid record`);
+                  }
+                }
+              } catch (error) {
+                results.stats.corruptedRecords++;
+                results.errors.push(`Line ${lineNumber + 1}: Invalid JSON - ${error.message}`);
+                if (verbose) {
+                  console.log(`❌ Line ${lineNumber + 1}: Corrupted JSON`);
+                }
+              }
+            }
+            lineNumber++;
+          }
+          offset += bytesRead;
+        }
+
+        // Process last line if it doesn't end with \n
+        if (lineBuffer.trim() !== '') {
+          results.warnings.push(`Line ${lineNumber + 1}: File doesn't end with newline`);
+        }
+      } finally {
+        await fd.close();
+      }
+    } catch (error) {
+      results.errors.push(`Error reading file: ${error.message}`);
+    }
+    return results;
+  }
+
+  /**
+   * Validates index consistency
+   */
+  async validateIndexes(verbose = false) {
+    const results = {
+      errors: [],
+      warnings: [],
+      stats: {
+        missingIndexes: 0,
+        orphanedIndexes: 0
+      }
+    };
+    const indexData = this.indexManager.serialize();
+    const validOffsets = new Set();
+
+    // Collect all valid offsets
+    for (let i = 0; i < this.indexManager.offsets.length; i++) {
+      if (this.indexManager.offsets[i] !== null) {
+        validOffsets.add(i);
+      }
+    }
+
+    // Check each index
+    for (const [field, fieldIndexData] of Object.entries(indexData.indexes)) {
+      if (verbose) {
+        console.log(`🔍 Checking index: ${field}`);
+      }
+      for (const [value, offsetArray] of Object.entries(fieldIndexData.values)) {
+        for (const offsetIndex of offsetArray) {
+          if (!validOffsets.has(offsetIndex)) {
+            results.stats.orphanedIndexes++;
+            results.errors.push(`Orphaned index: ${field}=${value} points to non-existent offset ${offsetIndex}`);
+          }
+        }
+      }
+    }
+
+    // Check if there are valid records without index
+    for (const offsetIndex of validOffsets) {
+      let hasIndex = false;
+      for (const [field, index] of Object.entries(this.indexManager.indexes)) {
+        for (const [value, offsetSet] of index.values.entries()) {
+          if (offsetSet.has(offsetIndex)) {
+            hasIndex = true;
+            break;
+          }
+        }
+        if (hasIndex) break;
+      }
+      if (!hasIndex) {
+        results.stats.missingIndexes++;
+        results.warnings.push(`Record at offset ${offsetIndex} is not indexed`);
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Validates offset consistency
+   */
+  async validateOffsets(verbose = false) {
+    const results = {
+      errors: [],
+      warnings: []
+    };
+    const stats = await this.fileHandler.getStats();
+    const fileSize = stats.size;
+
+    // Check if offsets are valid
+    for (let i = 0; i < this.indexManager.offsets.length; i++) {
+      const offset = this.indexManager.offsets[i];
+      if (offset !== null) {
+        if (offset < 0) {
+          results.errors.push(`Offset ${i}: Negative value (${offset})`);
+        } else if (offset >= fileSize) {
+          results.errors.push(`Offset ${i}: Out of file bounds (${offset} >= ${fileSize})`);
+        }
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Rebuilds indexes from the data file
+   */
+  async rebuildIndexes(verbose = false) {
+    if (verbose) {
+      console.log('🔧 Rebuilding indexes...');
+    }
+
+    // Store the configured indexes before clearing
+    const configuredIndexes = this.indexManager.indexes;
+
+    // Clear current indexes but preserve configuration
+    this.indexManager.clear();
+
+    // Restore the configured indexes
+    for (const [field, indexConfig] of Object.entries(configuredIndexes)) {
+      this.indexManager.indexes[field] = {
+        type: indexConfig.type,
+        values: new Map()
+      };
+    }
+    try {
+      const fd = await _fs.promises.open(this.fileHandler.filePath, 'r');
+      let lineNumber = 0;
+      let offset = 0;
+      const buffer = Buffer.alloc(8192);
+      let lineBuffer = '';
+      try {
+        while (true) {
+          const {
+            bytesRead
+          } = await fd.read(buffer, 0, buffer.length, offset);
+          if (bytesRead === 0) break;
+          const chunk = buffer.toString('utf8', 0, bytesRead);
+          lineBuffer += chunk;
+
+          // Process complete lines
+          let newlineIndex;
+          while ((newlineIndex = lineBuffer.indexOf('\n')) !== -1) {
+            const line = lineBuffer.substring(0, newlineIndex);
+            lineBuffer = lineBuffer.substring(newlineIndex + 1);
+            if (line.trim() !== '') {
+              try {
+                const record = JSON.parse(line);
+
+                // Only index non-deleted records
+                if (!record._deleted) {
+                  record._offset = offset;
+                  this.indexManager.addRecord(record, lineNumber);
+                  if (verbose) {
+                    console.log(`✅ Reindexed: line ${lineNumber + 1}`);
+                  }
+                }
+              } catch (error) {
+                if (verbose) {
+                  console.log(`⚠️ Line ${lineNumber + 1}: Ignored (invalid JSON)`);
+                }
+              }
+            }
+            lineNumber++;
+            offset += this.fileHandler.getByteLength(line + '\n');
+          }
+        }
+      } finally {
+        await fd.close();
+      }
+
+      // Save rebuilt indexes
+      await this.fileHandler.writeIndex(this.indexManager.serialize());
+      if (verbose) {
+        console.log('✅ Indexes rebuilt successfully');
+      }
+      return true;
+    } catch (error) {
+      if (verbose) {
+        console.error('❌ Error rebuilding indexes:', error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Exports detailed statistics
+   */
+  async exportStats() {
+    const stats = await this.fileHandler.getStats();
+    const indexStats = this.indexManager.getStats();
+    const integrityResults = await this.validateIntegrity({
+      verbose: false
+    });
+    return {
+      file: {
+        path: this.fileHandler.filePath,
+        size: stats.size,
+        created: stats.created,
+        modified: stats.modified
+      },
+      indexes: indexStats,
+      integrity: integrityResults,
+      summary: {
+        totalRecords: indexStats.recordCount,
+        fileSize: stats.size,
+        isValid: integrityResults.isValid,
+        errorCount: integrityResults.errors.length,
+        warningCount: integrityResults.warnings.length
+      }
+    };
+  }
+}
+var _default = exports.default = IntegrityChecker;