jexidb 2.0.2 → 2.0.3

package/docs/MIGRATION.md

@@ -0,0 +1,295 @@
+# JexiDB Migration Guide
+
+## Migrating from Other Databases
+
+### From JSON Files
+
+```javascript
+const fs = require('fs');
+const { Database } = require('jexidb');
+
+async function migrateFromJSON(jsonFilePath, dbPath) {
+  // Read JSON file
+  const jsonData = JSON.parse(fs.readFileSync(jsonFilePath, 'utf8'));
+  
+  // Create database
+  const db = new Database(dbPath);
+  await db.init();
+  
+  // Insert data
+  if (Array.isArray(jsonData)) {
+    await db.insertMany(jsonData);
+  } else {
+    await db.insert(jsonData);
+  }
+  
+  await db.save();
+  await db.destroy();
+  
+  console.log('Migration completed successfully');
+}
+
+// Usage
+migrateFromJSON('./old-data.json', './new-database.jsonl');
+```
+
+### From SQLite
+
+```javascript
+const sqlite3 = require('sqlite3');
+const { Database } = require('jexidb');
+
+async function migrateFromSQLite(sqlitePath, dbPath) {
+  const db = new sqlite3.Database(sqlitePath);
+  const database = new Database(dbPath);
+  await database.init();
+  
+  return new Promise((resolve, reject) => {
+    db.all('SELECT * FROM users', async (err, rows) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      
+      try {
+      await database.insertMany(rows);
+      await database.save();
+      await database.destroy();
+        console.log('Migration completed successfully');
+        resolve();
+      } catch (error) {
+        reject(error);
+      }
+    });
+  });
+}
+```
+
+### From MongoDB (Local)
+
+```javascript
+const { MongoClient } = require('mongodb');
+const { Database } = require('jexidb');
+
+async function migrateFromMongoDB(mongoUri, collectionName, dbPath) {
+  const client = new MongoClient(mongoUri);
+  await client.connect();
+  
+  const db = client.db();
+  const collection = db.collection(collectionName);
+  
+  const database = new Database(dbPath);
+  await database.init();
+  
+  const cursor = collection.find({});
+  const batch = [];
+  
+  while (await cursor.hasNext()) {
+    const doc = await cursor.next();
+    batch.push(doc);
+    
+    if (batch.length >= 1000) {
+      await database.insertMany(batch);
+      batch.length = 0;
+    }
+  }
+  
+  if (batch.length > 0) {
+    await database.insertMany(batch);
+  }
+  
+  await database.save();
+  await database.destroy();
+  await client.close();
+  
+  console.log('Migration completed successfully');
+}
+```
+
+## Data Transformation
+
+### Converting Data Types
+
+```javascript
+function transformData(oldData) {
+  return oldData.map(record => ({
+    id: record._id || record.id,
+    name: record.name || record.fullName,
+    email: record.email?.toLowerCase(),
+    age: parseInt(record.age) || 0,
+    metadata: {
+      created: record.createdAt || new Date().toISOString(),
+      updated: record.updatedAt || new Date().toISOString()
+    }
+  }));
+}
+
+// Usage
+const transformedData = transformData(oldData);
+await db.insertMany(transformedData);
+```
+
+### Handling Different Schemas
+
+```javascript
+function normalizeSchema(records) {
+  return records.map(record => {
+    const normalized = {
+      id: record.id || record._id || record.userId,
+      name: record.name || record.fullName || record.userName,
+      email: record.email || record.emailAddress,
+      age: record.age || record.userAge || 0
+    };
+    
+    // Add optional fields
+    if (record.metadata) {
+      normalized.metadata = record.metadata;
+    }
+    
+    if (record.tags) {
+      normalized.tags = Array.isArray(record.tags) ? record.tags : [record.tags];
+    }
+    
+    return normalized;
+  });
+}
+```
+
+## Index Migration
+
+### Creating Indexes for Existing Data
+
+```javascript
+async function createIndexesForExistingData(dbPath) {
+  const db = new Database(dbPath, {
+    indexes: {
+      id: true,
+      email: true,
+      'metadata.created': true
+    }
+  });
+  
+  await db.init();
+  
+  // Rebuild indexes for existing data
+  await db.rebuildIndexes({ verbose: true });
+  
+  await db.save();
+  await db.destroy();
+  
+  console.log('Indexes created successfully');
+}
+```
+
+## Validation and Verification
+
+### Verify Migration Success
+
+```javascript
+async function verifyMigration(originalData, dbPath) {
+  const db = new Database(dbPath);
+  await db.init();
+  
+  // Check record count
+  const count = await db.count();
+  console.log(`Original: ${originalData.length}, Migrated: ${count}`);
+  
+  // Check data integrity
+  const integrity = await db.validateIntegrity({ verbose: true });
+  console.log('Integrity check:', integrity.isValid ? 'PASSED' : 'FAILED');
+  
+  // Sample verification
+  const sample = await db.find({}, { limit: 5 });
+  console.log('Sample records:', sample);
+  
+  await db.destroy();
+}
+```
+
+## Performance Considerations
+
+### Large Dataset Migration
+
+```javascript
+async function migrateLargeDataset(data, dbPath, batchSize = 1000) {
+  const db = new Database(dbPath, {
+    autoSave: false, // Disable auto-save for better performance
+    backgroundMaintenance: false
+  });
+  
+  await db.init();
+  
+  for (let i = 0; i < data.length; i += batchSize) {
+    const batch = data.slice(i, i + batchSize);
+    await db.insertMany(batch);
+    
+    // Progress indicator
+    if (i % 10000 === 0) {
+      console.log(`Migrated ${i} records...`);
+    }
+  }
+  
+  await db.save();
+  await db.destroy();
+  
+  console.log('Large dataset migration completed');
+}
+```
+
+## Rollback Strategy
+
+### Backup Before Migration
+
+```javascript
+const fs = require('fs');
+
+async function backupBeforeMigration(originalPath) {
+  const backupPath = `${originalPath}.backup.${Date.now()}`;
+  await fs.copyFile(originalPath, backupPath);
+  console.log(`Backup created: ${backupPath}`);
+  return backupPath;
+}
+
+async function rollbackMigration(backupPath, currentPath) {
+  await fs.copyFile(backupPath, currentPath);
+  console.log('Rollback completed');
+}
+```
+
+## Migration Checklist
+
+- [ ] **Backup original data**
+- [ ] **Test migration with sample data**
+- [ ] **Verify data integrity**
+- [ ] **Check performance characteristics**
+- [ ] **Update application code**
+- [ ] **Test application functionality**
+- [ ] **Monitor for issues**
+- [ ] **Clean up old data** (after verification)
+
+## Common Migration Issues
+
+### 1. Data Type Mismatches
+
+**Issue:** Different data types between source and target
+**Solution:** Implement data transformation functions
+
+### 2. Missing Required Fields
+
+**Issue:** Target schema requires fields not in source
+**Solution:** Provide default values or skip records
+
+### 3. Duplicate Records
+
+**Issue:** Multiple records with same ID
+**Solution:** Use upsert operations or deduplication
+
+### 4. Performance Issues
+
+**Issue:** Migration taking too long
+**Solution:** Use batch operations and disable auto-save
+
+### 5. Memory Issues
+
+**Issue:** Running out of memory during migration
+**Solution:** Process data in smaller batches

package/docs/README.md

@@ -0,0 +1,184 @@
+# JexiDB - High-Performance Local JSONL Database
+
+[![Tests](https://img.shields.io/badge/tests-71%2F71%20passing-brightgreen)](https://github.com/jexidb/jexidb)
+[![Performance](https://img.shields.io/badge/performance-10k%20ops%2Fsec-brightgreen)](https://github.com/jexidb/jexidb)
+[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
+
+JexiDB is a high-performance, local JSONL database with intelligent optimizations, real compression, and comprehensive error handling. Perfect for Electron apps, Node.js applications, and any scenario requiring fast local data storage.
+
+## ✨ Features
+
+- 🚀 **High Performance**: 10,000+ ops/sec for bulk operations
+- 🧠 **Intelligent Optimizations**: Adaptive mode switching and query optimization
+- 📦 **Real Compression**: LZ4 and Gzip compression with automatic fallback
+- 🛡️ **Error Recovery**: Comprehensive error handling with automatic recovery
+- 🔍 **Advanced Queries**: MongoDB-style query operators and nested field support
+- 📊 **Performance Monitoring**: Built-in statistics and optimization recommendations
+- 🎯 **Zero Dependencies**: Pure JavaScript implementation
+- 🔧 **Production Ready**: 100% test coverage and comprehensive documentation
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+npm install jexidb
+```
+
+### Basic Usage
+
+```javascript
+const { Database } = require('jexidb');
+
+// Create database
+const db = new Database('./data/users.jsonl', {
+  indexes: { id: true, email: true }
+});
+
+// Initialize
+await db.init();
+
+// Insert data
+await db.insert({ id: '1', name: 'John', email: 'john@example.com' });
+
+// Find data
+const user = await db.findOne({ id: '1' });
+const users = await db.find({ name: { $regex: /john/i } });
+
+// Update data
+await db.update({ id: '1' }, { name: 'John Updated' });
+
+// Delete data
+await db.delete({ id: '1' });
+
+// Save and cleanup
+await db.save();
+await db.destroy();
+```
+
+## 📚 Documentation
+
+- [API Reference](docs/API.md) - Complete API documentation
+- [Usage Examples](docs/EXAMPLES.md) - Practical examples and patterns
+- [Performance Guide](docs/PERFORMANCE.md) - Optimization strategies
+- [Migration Guide](docs/MIGRATION.md) - Migrating from other databases
+
+## 🏗️ Architecture
+
+JexiDB is built with a modular architecture:
+
+- **Database**: Main database class with CRUD operations
+- **FileHandler**: Efficient file I/O with batch operations
+- **IndexManager**: Intelligent indexing with multiple strategies
+- **QueryOptimizer**: Query optimization and execution planning
+- **CompressionManager**: Real compression with LZ4 and Gzip
+- **CacheManager**: Intelligent caching with adaptive eviction
+- **ErrorHandler**: Comprehensive error recovery and logging
+- **BackgroundMaintenance**: Non-blocking maintenance operations
+
+## 🎯 Performance
+
+### Benchmarks
+
+| Operation | Throughput | Latency |
+|-----------|------------|---------|
+| Bulk Insert | 10,000 ops/sec | <1ms |
+| Single Insert | 1,000 ops/sec | <5ms |
+| Indexed Query | 5,000 ops/sec | <2ms |
+| Update | 1,000 ops/sec | <10ms |
+| Delete | 1,000 ops/sec | <10ms |
+
+### Compression
+
+- **LZ4**: 30-50% size reduction, very fast
+- **Gzip**: 20-40% size reduction, good compression
+- **Automatic**: Age-based compression strategy
+
+## 🔧 Configuration
+
+```javascript
+const db = new Database('./data.jsonl', {
+  // Indexes
+  indexes: {
+    id: true,
+    email: true,
+    'metadata.created': true
+  },
+  
+  // Behavior
+  markDeleted: true,
+  autoSave: true,
+  validateOnInit: false,
+  backgroundMaintenance: true,
+  
+  // Performance
+  cache: {
+    maxSize: 1000,
+    ttl: 300000
+  },
+  
+  // Compression
+  compression: {
+    hot: { type: 'none', threshold: 7 },
+    warm: { type: 'lz4', threshold: 30 },
+    cold: { type: 'gzip', threshold: Infinity }
+  },
+  
+  // Error handling
+  errorHandler: {
+    logLevel: 'info',
+    enableRecovery: true
+  }
+});
+```
+
+## 🧪 Testing
+
+```bash
+# Run all tests
+npm test
+
+# Run benchmarks
+npm run benchmark-performance
+npm run benchmark-compression
+
+# Generate test data
+npm run generate-test-db
+```
+
+## 📊 Monitoring
+
+```javascript
+// Get comprehensive statistics
+const stats = await db.getStats();
+console.log(`Records: ${stats.recordCount}`);
+console.log(`File size: ${stats.fileSize} bytes`);
+console.log(`Cache hit rate: ${stats.cacheStats.hitRate}%`);
+
+// Get optimization recommendations
+const recommendations = db.getOptimizationRecommendations();
+console.log('Optimization tips:', recommendations);
+```
+
+## 🤝 Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Add tests for new functionality
+5. Run the test suite
+6. Submit a pull request
+
+## 📄 License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## 🙏 Acknowledgments
+
+- Inspired by JexiDB but with significant improvements
+- Built with performance and reliability in mind
+- Designed for real-world production use
+
+---
+
+**JexiDB** - Fast, reliable, and intelligent local database storage.

package/examples/auto-save-example.js

@@ -0,0 +1,158 @@
+/**
+ * Auto-Save Example - Demonstrates JexiDB's intelligent auto-save features
+ * 
+ * This example shows:
+ * - Auto-save based on record count threshold
+ * - Auto-save based on time interval
+ * - Event handling for buffer operations
+ * - Buffer status monitoring
+ * - Performance configuration
+ */
+
+import Database from '../src/index.js';
+import path from 'path';
+
+async function autoSaveExample() {
+  console.log('🚀 Starting Auto-Save Example...\n');
+  
+  // Create database with intelligent auto-save configuration
+  const dbPath = path.join(process.cwd(), 'auto-save-example.jdb');
+  const db = new Database(dbPath, {
+    indexes: { category: 'string', priority: 'number' },
+    create: true,
+    clear: true,
+    
+    // Auto-save configuration
+    autoSave: true,
+    autoSaveThreshold: 10, // Flush every 10 records
+    autoSaveInterval: 3000, // Flush every 3 seconds
+    forceSaveOnClose: true,
+    
+    // Performance configuration
+    batchSize: 20,
+    adaptiveBatchSize: true,
+    minBatchSize: 5,
+    maxBatchSize: 50
+  });
+
+  // Set up event listeners
+  db.on('buffer-flush', (count) => {
+    console.log(`📤 Buffer flushed: ${count} records`);
+  });
+
+  db.on('buffer-full', () => {
+    console.log('⚠️  Buffer reached threshold, flushing...');
+  });
+
+  db.on('auto-save-timer', () => {
+    console.log('⏰ Auto-save timer triggered');
+  });
+
+  db.on('save-complete', () => {
+    console.log('✅ Save completed');
+  });
+
+  db.on('close-save-complete', () => {
+    console.log('🔒 Database closed with final save');
+  });
+
+  db.on('performance-configured', (config) => {
+    console.log('⚙️  Performance reconfigured:', config);
+  });
+
+  await db.init();
+  console.log('✅ Database initialized\n');
+
+  // Show initial buffer status
+  console.log('📊 Initial buffer status:');
+  console.log(db.getBufferStatus());
+  console.log('');
+
+  // Insert records to demonstrate auto-save
+  console.log('📝 Inserting records...');
+  
+  for (let i = 1; i <= 25; i++) {
+    const record = {
+      id: `item-${i}`,
+      name: `Item ${i}`,
+      category: i % 3 === 0 ? 'high' : i % 2 === 0 ? 'medium' : 'low',
+      priority: i % 5 + 1,
+      description: `This is item number ${i}`
+    };
+
+    await db.insert(record);
+    console.log(`  Inserted: ${record.name} (${record.category} priority)`);
+    
+    // Show buffer status every 5 records
+    if (i % 5 === 0) {
+      const status = db.getBufferStatus();
+      console.log(`  Buffer: ${status.pendingCount}/${status.bufferSize} records pending`);
+    }
+    
+    // Small delay to demonstrate time-based auto-save
+    await new Promise(resolve => setTimeout(resolve, 500));
+  }
+
+  console.log('\n📊 Final buffer status:');
+  console.log(db.getBufferStatus());
+  console.log('');
+
+  // Demonstrate performance configuration
+  console.log('⚙️  Reconfiguring performance...');
+  db.configurePerformance({
+    batchSize: 15,
+    autoSaveThreshold: 8,
+    autoSaveInterval: 2000
+  });
+
+  console.log('📊 New performance config:');
+  console.log(db.getPerformanceConfig());
+  console.log('');
+
+  // Insert more records with new configuration
+  console.log('📝 Inserting more records with new config...');
+  
+  for (let i = 26; i <= 35; i++) {
+    const record = {
+      id: `item-${i}`,
+      name: `Item ${i}`,
+      category: 'new',
+      priority: 10,
+      description: `New item ${i}`
+    };
+
+    await db.insert(record);
+    console.log(`  Inserted: ${record.name}`);
+    
+    await new Promise(resolve => setTimeout(resolve, 300));
+  }
+
+  // Query the database
+  console.log('\n🔍 Querying database...');
+  const highPriority = await db.find({ priority: { '>=': 8 } });
+  console.log(`Found ${highPriority.length} high priority items`);
+
+  const categories = await db.find({ category: 'high' });
+  console.log(`Found ${categories.length} high category items`);
+
+  // Show final statistics
+  console.log('\n📈 Final database statistics:');
+  const stats = db.stats;
+  console.log(`Total records: ${stats.recordCount}`);
+  console.log(`Buffer size: ${stats.insertionBufferSize}`);
+  console.log(`Auto-save enabled: ${stats.autoSave.enabled}`);
+  console.log(`Last flush: ${stats.autoSave.lastFlush ? new Date(stats.autoSave.lastFlush).toLocaleTimeString() : 'Never'}`);
+
+  // Close database (will trigger final save)
+  console.log('\n🔒 Closing database...');
+  await db.close();
+  
+  // Clean up
+  await db.destroy();
+  console.log('🧹 Cleanup completed');
+  
+  console.log('\n✅ Auto-Save Example completed successfully!');
+}
+
+// Run the example
+autoSaveExample().catch(console.error);

package/examples/cjs-usage.cjs

@@ -0,0 +1,82 @@
+// Example: Using Database with CommonJS
+// This file demonstrates how to use the library in CommonJS environments
+
+const Database = require('../dist/index.js').default;
+const { utils, OPERATORS } = require('../dist/index.js');
+
+async function exampleCJS() {
+  console.log('🚀 Database CommonJS Example');
+  console.log('========================\n');
+
+  // Create a new database
+  const db = new Database('./example-cjs.jsonl', {
+    indexes: {
+      id: 'number',
+      email: 'string',
+      age: 'number'
+    }
+  });
+
+  try {
+    // Initialize the database
+    await db.init();
+    console.log('✅ Database initialized');
+
+    // Insert some records
+    const user1 = await db.insert({
+      id: 1,
+      name: 'John Doe',
+      email: 'john@example.com',
+      age: 30,
+      city: 'New York'
+    });
+
+    const user2 = await db.insert({
+      id: 2,
+      name: 'Jane Smith',
+      email: 'jane@example.com',
+      age: 25,
+      city: 'Los Angeles'
+    });
+
+    console.log('✅ Records inserted:', user1.name, user2.name);
+
+    // Find records using different queries
+    const allUsers = await db.find({});
+    console.log('📋 All users:', allUsers.length);
+
+    const youngUsers = await db.find({ age: { '<': 30 } });
+    console.log('👶 Young users:', youngUsers.length);
+
+    const johnUser = await db.find({ email: 'john@example.com' });
+    console.log('👤 John user:', johnUser[0]?.name);
+
+    // Update a record
+    const updateResult = await db.update(
+      { id: 1 },
+      { age: 31, city: 'Boston' }
+    );
+    console.log('🔄 Updated records:', updateResult.updatedCount);
+
+    // Use utility functions
+    const stats = db.stats;
+    console.log('📊 Database stats:', {
+      recordCount: stats.recordCount,
+      indexedFields: stats.indexedFields
+    });
+
+    // Save the database
+    await db.save();
+    console.log('💾 Database saved');
+
+    // Close the database
+    await db.close();
+    console.log('🔒 Database closed');
+
+  } catch (error) {
+    console.error('❌ Error:', error.message);
+  }
+}
+
+// Run the example
+exampleCJS().catch(console.error);