jexidb 2.1.0 → 2.1.1

package/test/deserialize-corruption-fixes.test.js

@@ -0,0 +1,296 @@
+import { Database } from '../src/Database.mjs'
+import fs from 'fs'
+import path from 'path'
+
+describe('Deserialize Corruption Fixes', () => {
+  let testDir
+  let db
+
+  beforeEach(() => {
+    testDir = path.join(process.cwd(), 'test-files', 'deserialize-corruption')
+    fs.mkdirSync(testDir, { recursive: true })
+  })
+
+  afterEach(async () => {
+    if (db) {
+      await db.close()
+    }
+    // Clean up test files
+    if (fs.existsSync(testDir)) {
+      try {
+        fs.rmSync(testDir, { recursive: true, force: true })
+      } catch (error) {
+        console.warn('Could not clean up test directory:', testDir)
+      }
+    }
+  })
+
+  describe('Multiple JSON objects in same line', () => {
+    test('deserialize should recover from multiple JSON objects in same string', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test direct deserialization with multiple objects
+      const corruptedData = '{"name":"Test1","value":1}{"name":"Test2","value":2}'
+      
+      // Should recover the first object
+      const result = serializer.deserialize(corruptedData)
+      expect(result).toBeTruthy()
+      expect(result.name).toBe('Test1')
+      expect(result.value).toBe(1)
+    })
+
+    test('deserialize should recover from multiple JSON arrays in same string', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test direct deserialization with multiple arrays
+      const corruptedData = '["http://test1.com","Test1"]["http://test2.com","Test2"]'
+      
+      // Should recover the first array
+      const result = serializer.deserialize(corruptedData)
+      expect(result).toBeTruthy()
+      expect(Array.isArray(result)).toBe(true)
+    })
+
+    test('should handle deserialize with multiple objects gracefully', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test with multiple JSON objects in one string
+      const corruptedData = '{"name":"Test1","value":1}{"name":"Test2","value":2}'
+      
+      // Should recover the first object
+      const result = serializer.deserialize(corruptedData)
+      expect(result).toBeTruthy()
+      expect(result.name).toBe('Test1')
+      expect(result.value).toBe(1)
+    })
+
+    test('should handle deserialize with multiple arrays gracefully', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test with multiple JSON arrays in one string
+      const corruptedData = '["http://test1.com","Test1"]["http://test2.com","Test2"]'
+      
+      // Should recover the first array
+      const result = serializer.deserialize(corruptedData)
+      expect(result).toBeTruthy()
+      expect(Array.isArray(result)).toBe(true)
+    })
+
+    test('should handle JSON objects with braces inside string values', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test JSON object with braces inside string values
+      const dataWithBracesInString = '{"key": "value with { brace", "url": "http://example.com?param={value}"}'
+      
+      const result = serializer.deserialize(dataWithBracesInString)
+      expect(result).toBeTruthy()
+      expect(result.key).toBe('value with { brace')
+      expect(result.url).toBe('http://example.com?param={value}')
+    })
+
+    test('should handle JSON arrays with brackets inside string values', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test JSON array with brackets inside string values
+      const dataWithBracketsInString = '["item with [ bracket", "url with [param]"]'
+      
+      const result = serializer.deserialize(dataWithBracketsInString)
+      expect(result).toBeTruthy()
+      expect(Array.isArray(result)).toBe(true)
+      expect(result[0]).toBe('item with [ bracket')
+      expect(result[1]).toBe('url with [param]')
+    })
+
+    test('should handle JSON with escaped quotes and braces', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test JSON with escaped quotes and braces inside strings
+      const dataWithEscaped = '{"key": "value with \\" escaped quote and { brace", "nested": "text with } closing"}'
+      
+      const result = serializer.deserialize(dataWithEscaped)
+      expect(result).toBeTruthy()
+      expect(result.key).toBe('value with " escaped quote and { brace')
+      expect(result.nested).toBe('text with } closing')
+    })
+
+    test('should handle multiple JSON objects when first has braces in strings', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test multiple JSON objects where first has braces in string values
+      const corruptedData = '{"key": "value with { brace", "url": "http://example.com?param={value}"}{"name":"Test2","value":2}'
+      
+      // Should recover the first object correctly, ignoring braces inside strings
+      const result = serializer.deserialize(corruptedData)
+      expect(result).toBeTruthy()
+      expect(result.key).toBe('value with { brace')
+      expect(result.url).toBe('http://example.com?param={value}')
+      expect(result.name).toBeUndefined() // Should not include second object
+    })
+
+    test('should handle real-world URL array with special characters', async () => {
+      const Serializer = (await import('../src/Serializer.mjs')).default
+      const serializer = new Serializer({ debugMode: false })
+
+      // Test real-world scenario: URL array with special characters (similar to the original error)
+      const realWorldData = '["http://113.164.225.140:1935/live/quochoitvlive.stream_720p/playlist.m3u8?IMDSFULL","Quốc Hội","http://example.com?param={value}"]'
+      
+      const result = serializer.deserialize(realWorldData)
+      expect(result).toBeTruthy()
+      expect(Array.isArray(result)).toBe(true)
+      expect(result[0]).toBe('http://113.164.225.140:1935/live/quochoitvlive.stream_720p/playlist.m3u8?IMDSFULL')
+      expect(result[1]).toBe('Quốc Hội')
+      expect(result[2]).toBe('http://example.com?param={value}')
+    })
+  })
+
+  describe('walk() error handling', () => {
+    test('should continue processing after encountering corrupted line', async () => {
+      const dbPath = path.join(testDir, 'walk-corruption.jdb')
+      db = new Database(dbPath, { clear: true, create: true, debugMode: false })
+      await db.init()
+
+      // Insert multiple records
+      for (let i = 0; i < 10; i++) {
+        await db.insert({ name: `Test${i}`, value: i })
+      }
+      await db.save()
+
+      // Manually corrupt one line by replacing with completely invalid data
+      // This tests that walk() continues processing other valid records
+      // Note: Corrupting the file manually can affect offsets, so we expect some records to be skipped
+      const fileContent = fs.readFileSync(dbPath, 'utf8')
+      const lines = fileContent.split('\n').filter(l => l.trim())
+      
+      // Corrupt the 5th line - replace with invalid data
+      // The walk() should skip this and continue with other records
+      if (lines.length > 4) {
+        lines[4] = 'INVALID_LINE_WITH_NO_JSON'
+        const corruptedContent = lines.join('\n')
+        fs.writeFileSync(dbPath, corruptedContent, 'utf8')
+      }
+
+      // Walk should continue processing other records (skip the corrupted one)
+      let count = 0
+      const records = []
+      for await (const record of db.walk()) {
+        records.push(record)
+        count++
+      }
+
+      // Should process at least some records (walk() should not stop completely)
+      // Note: Manual file corruption can affect offsets, so we're lenient with the count
+      expect(count).toBeGreaterThan(0)
+      expect(count).toBeLessThan(11) // Should be less than 10 due to corrupted line
+      
+      // Most importantly: walk() should complete without throwing an unhandled error
+      // The fact that we got here means it handled the corruption gracefully
+    })
+
+    test('should log errors in debug mode', async () => {
+      const dbPath = path.join(testDir, 'walk-debug.jdb')
+      db = new Database(dbPath, { clear: true, create: true, debugMode: true })
+      await db.init()
+
+      await db.insert({ name: 'Test1', value: 1 })
+      await db.insert({ name: 'Test2', value: 2 })
+      await db.insert({ name: 'Test3', value: 3 })
+      await db.save()
+
+      // Manually corrupt one line with invalid JSON (not the first one to avoid offset issues)
+      const fileContent = fs.readFileSync(dbPath, 'utf8')
+      const lines = fileContent.split('\n').filter(l => l.trim())
+      // Corrupt middle line with invalid data (line 1, not 0, to avoid breaking offsets)
+      if (lines.length > 1) {
+        lines[1] = 'INVALID_JSON_NO_BRACES_OR_BRACKETS'
+        fs.writeFileSync(dbPath, lines.join('\n'), 'utf8')
+      }
+
+      // Capture console.warn to verify errors are logged in debug mode
+      const consoleSpy = jest.spyOn(console, 'warn').mockImplementation(() => {})
+
+      let count = 0
+      let walkCompleted = false
+      try {
+        for await (const record of db.walk()) {
+          count++
+        }
+        walkCompleted = true
+      } catch (error) {
+        // walk() should not throw unhandled errors - it should catch and log them
+        throw error
+      }
+
+      // Most importantly: walk() should complete without throwing an unhandled error
+      expect(walkCompleted).toBe(true)
+      
+      // If records were processed, verify we got some
+      // Note: Manual file corruption can affect offsets, so count may be 0
+      // The key is that walk() handled the corruption gracefully
+      if (count > 0) {
+        expect(count).toBeGreaterThan(0)
+        // If we processed records, we likely skipped the corrupted one
+        expect(count).toBeLessThan(4) // Should be less than 3 due to corrupted line
+      }
+
+      consoleSpy.mockRestore()
+    })
+  })
+
+  describe('save() waits for auto-flushes', () => {
+    test('save() should wait for auto-flushes before writing', async () => {
+      const dbPath = path.join(testDir, 'save-waits-flush.jdb')
+      db = new Database(dbPath, { clear: true, create: true })
+      await db.init()
+
+      const session = db.beginInsertSession({ batchSize: 10 })
+      
+      // Insert records to trigger auto-flushes
+      for (let i = 0; i < 50; i++) {
+        await session.add({ name: `Record ${i}`, value: i })
+      }
+
+      // Save should wait for all auto-flushes to complete
+      await db.save()
+
+      // Verify all data was saved
+      expect(db.length).toBe(50)
+      
+      // Verify data is accessible
+      const results = await db.find({ name: 'Record 0' })
+      expect(results.length).toBe(1)
+    })
+
+    test('save() should handle multiple active sessions', async () => {
+      const dbPath = path.join(testDir, 'save-multiple-sessions.jdb')
+      db = new Database(dbPath, { clear: true, create: true })
+      await db.init()
+
+      const session1 = db.beginInsertSession({ batchSize: 10 })
+      const session2 = db.beginInsertSession({ batchSize: 10 })
+      
+      // Insert records in both sessions
+      const promises = []
+      for (let i = 0; i < 25; i++) {
+        promises.push(session1.add({ name: `Session1-${i}`, value: i }))
+        promises.push(session2.add({ name: `Session2-${i}`, value: i }))
+      }
+
+      await Promise.all(promises)
+
+      // Save should wait for all auto-flushes from both sessions
+      await db.save()
+
+      // Verify all data was saved
+      expect(db.length).toBe(50)
+    })
+  })
+})
+

package/test/exists-method.test.js

@@ -0,0 +1,318 @@
+/**
+ * Comprehensive tests for exists() method
+ * Tests index-only existence checks with maximum performance
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
+import Database from '../src/Database.mjs';
+import fs from 'fs';
+import path from 'path';
+
+describe('exists() Method', () => {
+  let db;
+  const testDbPath = 'test-exists-method';
+
+  beforeEach(async () => {
+    // Clean up any existing test database
+    const files = [
+      testDbPath + '.jdb',
+      testDbPath + '.idx.jdb',
+      testDbPath + '.terms.jdb'
+    ];
+    for (const file of files) {
+      if (fs.existsSync(file)) {
+        fs.unlinkSync(file);
+      }
+    }
+
+    db = new Database(testDbPath, {
+      debugMode: false,
+      termMapping: true,
+      termMappingFields: ['nameTerms', 'tags'],
+      indexes: {
+        nameTerms: 'array:string',
+        tags: 'array:string',
+        group: 'string',
+        rating: 'number'
+      }
+    });
+
+    await db.init();
+
+    // Insert comprehensive test data
+    const testData = [
+      { id: 1, name: 'TV Câmara', nameTerms: ['tv', 'câmara'], tags: ['news', 'politics'], group: 'Brazil', rating: 4.5 },
+      { id: 2, name: 'TV Cultura', nameTerms: ['tv', 'cultura'], tags: ['culture', 'education'], group: 'Brazil', rating: 4.2 },
+      { id: 3, name: 'SBT', nameTerms: ['sbt'], tags: ['entertainment'], group: 'Brazil', rating: 3.8 },
+      { id: 4, name: 'Record News', nameTerms: ['record', 'news'], tags: ['news'], group: 'Brazil', rating: 4.0 },
+      { id: 5, name: 'CNN', nameTerms: ['cnn'], tags: ['news', 'international'], group: 'International', rating: 4.7 },
+      { id: 6, name: 'BBC', nameTerms: ['bbc'], tags: ['news', 'international'], group: 'International', rating: 4.6 },
+      { id: 7, name: 'TV Globo', nameTerms: ['tv', 'globo'], tags: ['entertainment', 'news'], group: 'Brazil', rating: 4.3 },
+      { id: 8, name: 'TV Record', nameTerms: ['tv', 'record'], tags: ['entertainment'], group: 'Brazil', rating: 3.9 },
+      { id: 9, name: 'Discovery', nameTerms: ['discovery'], tags: ['documentary', 'education'], group: 'International', rating: 4.4 },
+      { id: 10, name: 'National Geographic', nameTerms: ['national', 'geographic'], tags: ['documentary', 'nature'], group: 'International', rating: 4.8 }
+    ];
+
+    for (const record of testData) {
+      await db.insert(record);
+    }
+    
+    // Save to ensure indexes are persisted
+    await db.save();
+  });
+
+  afterEach(async () => {
+    if (db) {
+      await db.close();
+    }
+    
+    // Clean up test files
+    const files = [
+      testDbPath + '.jdb',
+      testDbPath + '.idx.jdb',
+      testDbPath + '.terms.jdb'
+    ];
+    for (const file of files) {
+      if (fs.existsSync(file)) {
+        fs.unlinkSync(file);
+      }
+    }
+  });
+
+  describe('Basic exists() - $in behavior (default)', () => {
+    it('should return true if term exists', async () => {
+      const exists = await db.exists('nameTerms', 'tv');
+      expect(exists).toBe(true);
+    });
+
+    it('should return false if term does not exist', async () => {
+      const exists = await db.exists('nameTerms', 'nonexistent');
+      expect(exists).toBe(false);
+    });
+
+    it('should return true if any term in array exists', async () => {
+      const exists = await db.exists('nameTerms', ['tv', 'nonexistent']);
+      expect(exists).toBe(true);
+    });
+
+    it('should return false if no terms in array exist', async () => {
+      const exists = await db.exists('nameTerms', ['nonexistent1', 'nonexistent2']);
+      expect(exists).toBe(false);
+    });
+
+    it('should match count() > 0 behavior', async () => {
+      const testTerms = ['tv', 'sbt', 'cnn', 'nonexistent'];
+      
+      for (const term of testTerms) {
+        const exists = await db.exists('nameTerms', term);
+        const count = await db.count({ nameTerms: term });
+        expect(exists).toBe(count > 0);
+      }
+    });
+  });
+
+  describe('exists() with $all option', () => {
+    it('should return true if all terms exist and have intersection', async () => {
+      // 'tv' and 'globo' both exist and have intersection (TV Globo)
+      const exists = await db.exists('nameTerms', ['tv', 'globo'], { $all: true });
+      expect(exists).toBe(true);
+    });
+
+    it('should return false if all terms exist but no intersection', async () => {
+      // 'sbt' and 'cnn' both exist but have no intersection
+      const exists = await db.exists('nameTerms', ['sbt', 'cnn'], { $all: true });
+      expect(exists).toBe(false);
+    });
+
+    it('should return false if any term does not exist', async () => {
+      const exists = await db.exists('nameTerms', ['tv', 'nonexistent'], { $all: true });
+      expect(exists).toBe(false);
+    });
+
+    it('should return true for single term with $all', async () => {
+      const exists = await db.exists('nameTerms', 'tv', { $all: true });
+      expect(exists).toBe(true);
+    });
+
+    it('should match count() with $all behavior', async () => {
+      const testCases = [
+        { terms: ['tv', 'globo'], shouldExist: true },
+        { terms: ['tv', 'cultura'], shouldExist: true },
+        { terms: ['sbt', 'cnn'], shouldExist: false },
+        { terms: ['tv', 'nonexistent'], shouldExist: false }
+      ];
+
+      for (const testCase of testCases) {
+        const exists = await db.exists('nameTerms', testCase.terms, { $all: true });
+        const count = await db.count({ nameTerms: { $all: testCase.terms } });
+        expect(exists).toBe(count > 0);
+        expect(exists).toBe(testCase.shouldExist);
+      }
+    });
+  });
+
+  describe('exists() with term mapping', () => {
+    it('should work with term mapping fields', async () => {
+      const exists = await db.exists('nameTerms', 'tv');
+      expect(exists).toBe(true);
+    });
+
+    it('should return false for unmapped terms', async () => {
+      // Term that was never inserted should not exist in term mapping
+      const exists = await db.exists('nameTerms', 'neverinserted');
+      expect(exists).toBe(false);
+    });
+
+    it('should work with $all and term mapping', async () => {
+      const exists = await db.exists('nameTerms', ['tv', 'globo'], { $all: true });
+      expect(exists).toBe(true);
+    });
+  });
+
+  describe('exists() with case-insensitive option', () => {
+    it('should find matches case-insensitively', async () => {
+      const exists = await db.exists('nameTerms', 'TV', { caseInsensitive: true });
+      expect(exists).toBe(true);
+    });
+
+    it('should work with case-insensitive and $all', async () => {
+      const exists = await db.exists('nameTerms', ['TV', 'GLOBO'], { 
+        $all: true, 
+        caseInsensitive: true 
+      });
+      expect(exists).toBe(true);
+    });
+  });
+
+  describe('exists() edge cases', () => {
+    it('should return false for non-indexed field', async () => {
+      const exists = await db.exists('name', 'TV Câmara');
+      expect(exists).toBe(false);
+    });
+
+    it('should return false for invalid fieldName', async () => {
+      const exists = await db.exists('', 'tv');
+      expect(exists).toBe(false);
+      
+      const exists2 = await db.exists(null, 'tv');
+      expect(exists2).toBe(false);
+    });
+
+    it('should return false for empty terms array', async () => {
+      const exists = await db.exists('nameTerms', []);
+      expect(exists).toBe(false);
+    });
+
+    it('should handle numeric fields', async () => {
+      const exists = await db.exists('rating', 4.5);
+      expect(exists).toBe(true);
+      
+      const exists2 = await db.exists('rating', 999);
+      expect(exists2).toBe(false);
+    });
+  });
+
+  describe('exists() performance - index-only (no disk I/O)', () => {
+    it('should be faster than count() for existence checks', async () => {
+      const testTerm = 'tv';
+      
+      // Warm up
+      await db.exists('nameTerms', testTerm);
+      await db.count({ nameTerms: testTerm });
+      
+      // Measure exists()
+      const startExists = Date.now();
+      for (let i = 0; i < 100; i++) {
+        await db.exists('nameTerms', testTerm);
+      }
+      const timeExists = Date.now() - startExists;
+      
+      // Measure count()
+      const startCount = Date.now();
+      for (let i = 0; i < 100; i++) {
+        await db.count({ nameTerms: testTerm });
+      }
+      const timeCount = Date.now() - startCount;
+      
+      // exists() should be faster (index-only, no disk I/O)
+      // Note: This is a basic check - actual performance depends on many factors
+      console.log(`exists(): ${timeExists}ms, count(): ${timeCount}ms`);
+      expect(timeExists).toBeLessThan(timeCount * 10); // exists() should be significantly faster
+    });
+
+    it('should work directly with indexManager (synchronous)', () => {
+      // Test that indexManager.exists() is synchronous and works
+      const exists = db.indexManager.exists('nameTerms', 'tv');
+      expect(exists).toBe(true);
+      
+      const notExists = db.indexManager.exists('nameTerms', 'nonexistent');
+      expect(notExists).toBe(false);
+    });
+  });
+
+  describe('exists() with multiple fields', () => {
+    it('should work with tags field', async () => {
+      const exists = await db.exists('tags', 'news');
+      expect(exists).toBe(true);
+      
+      const existsAll = await db.exists('tags', ['news', 'politics'], { $all: true });
+      expect(existsAll).toBe(true); // TV Câmara has both
+    });
+
+    it('should work with group field', async () => {
+      const exists = await db.exists('group', 'Brazil');
+      expect(exists).toBe(true);
+      
+      const exists2 = await db.exists('group', 'Nonexistent');
+      expect(exists2).toBe(false);
+    });
+  });
+
+  describe('exists() with excludes option', () => {
+    it('should return false if all matches are excluded', async () => {
+      // 'tv' exists, but if we exclude all records that have 'tv', should return false
+      // Actually, this tests if 'tv' exists but NOT 'globo' (TV Globo has both)
+      const exists = await db.exists('nameTerms', 'tv', { excludes: ['globo'] });
+      expect(exists).toBe(true); // TV Câmara, TV Cultura, TV Record have 'tv' but not 'globo'
+    });
+
+    it('should return true if some matches are not excluded', async () => {
+      // Records with 'tv' but not 'cultura'
+      const exists = await db.exists('nameTerms', 'tv', { excludes: ['cultura'] });
+      expect(exists).toBe(true); // TV Câmara, TV Globo, TV Record have 'tv' but not 'cultura'
+    });
+
+    it('should work with $all and excludes', async () => {
+      // Records with both 'tv' and 'globo' but not 'news'
+      const exists = await db.exists('nameTerms', ['tv', 'globo'], { 
+        $all: true, 
+        excludes: ['news'] 
+      });
+      expect(exists).toBe(true); // TV Globo has 'tv' and 'globo' but tags have 'news', not nameTerms
+    });
+
+    it('should return false if excludes remove all candidates', async () => {
+      // Try to find 'tv' but exclude something that all 'tv' records have
+      // This is tricky - let's use a term that doesn't exist to test the logic
+      const exists = await db.exists('nameTerms', 'nonexistent', { excludes: ['also-nonexistent'] });
+      expect(exists).toBe(false);
+    });
+
+    it('should work with excludes and case-insensitive', async () => {
+      const exists = await db.exists('nameTerms', 'TV', { 
+        caseInsensitive: true, 
+        excludes: ['globo'] 
+      });
+      expect(exists).toBe(true);
+    });
+
+    it('should work with multiple exclude terms', async () => {
+      // Records with 'tv' but not 'globo' and not 'cultura'
+      const exists = await db.exists('nameTerms', 'tv', { 
+        excludes: ['globo', 'cultura'] 
+      });
+      expect(exists).toBe(true); // TV Câmara, TV Record have 'tv' but not 'globo' or 'cultura'
+    });
+  });
+});
+