jexidb 2.0.3 → 2.1.1

package/test/explicit-indexes-comparison.test.js

@@ -0,0 +1,219 @@
+import { Database } from '../src/Database.mjs'
+import fs from 'fs'
+
+describe('Explicit indexes with comparison operators', () => {
+  let testDbPath
+  let testIdxPath
+
+  beforeEach(() => {
+    const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`
+    testDbPath = `test-explicit-index-${uniqueSuffix}.jdb`
+    testIdxPath = testDbPath.replace('.jdb', '.idx.jdb')
+  })
+
+  afterEach(() => {
+    for (const filePath of [testDbPath, testIdxPath]) {
+      if (filePath && fs.existsSync(filePath)) {
+        try {
+          fs.unlinkSync(filePath)
+        } catch (error) {
+          console.warn(`⚠️  Failed to delete ${filePath}: ${error.message}`)
+        }
+      }
+    }
+  })
+
+  test('should reuse persisted index configuration when explicit indexes array is provided', async () => {
+    const seedDb = new Database(testDbPath, {
+      indexes: {
+        channel: 'string',
+        end: 'number',
+        start: 'number',
+        terms: 'array:string'
+      },
+      debugMode: false
+    })
+    await seedDb.init()
+
+    const now = Math.floor(Date.now() / 1000)
+
+    await seedDb.insert({
+      id: '1',
+      channel: 'Example Channel',
+      end: now + 120
+    })
+
+    await seedDb.insert({
+      id: '2',
+      channel: 'Example Channel',
+      end: now - 120
+    })
+
+    await seedDb.insert({
+      id: '3',
+      channel: 'Other Channel',
+      start: now + 1800,
+      end: now + 3600,
+      terms: ['other']
+    })
+
+    await seedDb.save()
+    await seedDb.close()
+
+    const reopenedDb = new Database(testDbPath, {
+      create: false,
+      indexes: ['channel', 'end'],
+      debugMode: false
+    })
+
+    await reopenedDb.init()
+
+    const count = await reopenedDb.count({
+      channel: 'Example Channel',
+      end: { $gt: now }
+    })
+    expect(count).toBe(1)
+
+    const results = await reopenedDb.find({
+      channel: 'Example Channel',
+      end: { $gt: now }
+    })
+    expect(results.map(record => record.id)).toEqual(['1'])
+
+    await reopenedDb.destroy()
+  })
+
+  test('should return results for equality mixed with comparison operators after reopening with persisted indexes', async () => {
+    const seedDb = new Database(testDbPath, {
+      indexes: {
+        channel: 'string',
+        start: 'number',
+        end: 'number',
+        terms: 'array:string'
+      },
+      indexedQueryMode: 'permissive',
+      debugMode: false
+    })
+    await seedDb.init()
+
+    const now = Math.floor(Date.now() / 1000)
+
+    await seedDb.insert({
+      id: 'future-1',
+      channel: 'São Paulo|SP  Disney Channel',
+      start: now + 600,
+      end: now + 3600,
+      terms: ['disney', 'kids']
+    })
+    await seedDb.insert({
+      id: 'past-1',
+      channel: 'São Paulo|SP  Disney Channel',
+      start: now - 3600,
+      end: now - 300,
+      terms: ['disney', 'classic']
+    })
+    await seedDb.insert({
+      id: 'present-1',
+      channel: 'São Paulo|SP  Disney Channel',
+      start: now - 60,
+      end: now + 60,
+      terms: ['disney', 'now']
+    })
+    await seedDb.insert({
+      id: 'other-1',
+      channel: 'Other Channel',
+      start: now + 600,
+      end: now + 3600,
+      terms: ['other']
+    })
+
+    await seedDb.save()
+    await seedDb.close()
+
+    const reopenedDb = new Database(testDbPath, {
+      create: false,
+      indexes: {
+        channel: 'string',
+        start: 'number',
+        end: 'number',
+        terms: 'array:string'
+      },
+      indexedQueryMode: 'permissive',
+      debugMode: false
+    })
+
+    await reopenedDb.init()
+
+    // Simulate regression: channel index line numbers loaded as strings while numeric index stays numeric
+    const reopenedChannelIndex = reopenedDb.indexManager.index.data.channel
+    for (const term in reopenedChannelIndex) {
+      const entry = reopenedChannelIndex[term]
+      if (entry?.set instanceof Set) {
+        reopenedChannelIndex[term].set = new Set(Array.from(entry.set).map(value => String(value)))
+      }
+    }
+
+    const disneyChannel = 'São Paulo|SP  Disney Channel'
+
+    const equalityCount = await reopenedDb.count({ channel: disneyChannel })
+    expect(equalityCount).toBe(3)
+
+    // Ensure persisted channel index still contains stringified line numbers
+    const storedChannelLineTypes = Object.values(reopenedChannelIndex).flatMap(entry => {
+      if (entry?.set instanceof Set) {
+        return Array.from(entry.set).map(value => typeof value)
+      }
+      return []
+    })
+    expect(storedChannelLineTypes.every(type => type === 'string')).toBe(true)
+
+    // Normalization should convert returned line numbers to numeric form
+    const channelLineTypes = Array.from(reopenedDb.indexManager.query({ channel: disneyChannel })).map(value => typeof value)
+    expect(channelLineTypes.every(type => type === 'number')).toBe(true)
+
+    const endLineTypes = Array.from(reopenedDb.indexManager.query({ end: { $gt: now } })).map(value => typeof value)
+    expect(endLineTypes.every(type => type === 'number')).toBe(true)
+
+    const operators = [
+      { criteria: { channel: disneyChannel, end: { $gt: now } }, expectedIds: ['future-1', 'present-1'] },
+      { criteria: { channel: disneyChannel, end: { '>': now } }, expectedIds: ['future-1', 'present-1'] },
+      { criteria: { channel: disneyChannel, end: { $gte: now } }, expectedIds: ['future-1', 'present-1'] },
+      { criteria: { channel: disneyChannel, end: { $lt: now } }, expectedIds: ['past-1'] },
+      { criteria: { channel: disneyChannel, end: { '<': now } }, expectedIds: ['past-1'] },
+      { criteria: { channel: disneyChannel, end: { $lte: now } }, expectedIds: ['past-1'] },
+      { criteria: { channel: disneyChannel, end: { '<=': now } }, expectedIds: ['past-1'] },
+      { criteria: { channel: disneyChannel, start: { $lte: now } }, expectedIds: ['past-1', 'present-1'] },
+      { criteria: { channel: disneyChannel, start: { '<': now } }, expectedIds: ['past-1', 'present-1'] }
+    ]
+
+    for (const { criteria, expectedIds } of operators) {
+      const count = await reopenedDb.count(criteria)
+      expect(count).toBe(expectedIds.length)
+
+      const results = await reopenedDb.find(criteria)
+      expect(results.map(record => record.id).sort()).toEqual([...expectedIds].sort())
+    }
+
+    const termsComparisonCount = await reopenedDb.count({
+      terms: { $in: ['disney'] },
+      end: { $gt: now }
+    })
+    expect(termsComparisonCount).toBe(2)
+
+    const termsComparisonIds = (await reopenedDb.find({
+      terms: { $in: ['disney'] },
+      end: { $gt: now }
+    })).map(record => record.id).sort()
+    expect(termsComparisonIds).toEqual(['future-1', 'present-1'])
+
+    const integrationCheckCount = await reopenedDb.count({
+      channel: disneyChannel,
+      end: { $gt: now },
+      start: { $lte: now + 3600 }
+    })
+    expect(integrationCheckCount).toBe(2)
+
+    await reopenedDb.destroy()
+  })
+})
+

package/test/filehandler-non-adjacent-ranges-bug.test.js

@@ -0,0 +1,175 @@
+import { Database } from '../src/Database.mjs'
+import fs from 'fs'
+import path from 'path'
+
+describe('FileHandler readGroupedRange Bug - Non-adjacent Ranges', () => {
+  let testDir
+  let db
+
+  beforeEach(() => {
+    testDir = path.join(process.cwd(), 'test-files', 'filehandler-bug')
+    fs.mkdirSync(testDir, { recursive: true })
+  })
+
+  afterEach(async () => {
+    if (db) {
+      await db.destroy()
+    }
+    if (fs.existsSync(testDir)) {
+      try {
+        fs.rmSync(testDir, { recursive: true, force: true })
+      } catch (error) {
+        // Ignore cleanup errors
+      }
+    }
+  })
+
+  test('should detect deserialization error when readGroupedRange extracts incomplete lines', async () => {
+    // This test directly reproduces the bug where substring() extracts incomplete lines
+    // from non-adjacent ranges in the same buffer
+    
+    const dbPath = path.join(testDir, 'bug-reproduction.jdb')
+    
+    if (fs.existsSync(dbPath)) {
+      fs.unlinkSync(dbPath)
+    }
+    if (fs.existsSync(dbPath + '.idx')) {
+      fs.unlinkSync(dbPath + '.idx')
+    }
+
+    db = new Database(dbPath, {
+      clear: true,
+      create: true,
+      fields: {
+        id: 'number',
+        url: 'string',
+        name: 'string',
+        logo: 'string',
+        category: 'string',
+        nameTerms: 'array:string'
+      },
+      indexes: {
+        nameTerms: 'array:string'
+      },
+      debugMode: false
+    })
+
+    await db.init()
+
+    // Create a scenario that will force non-adjacent ranges to be grouped together
+    // Insert many records, with only a few matching the query
+    // The matching records should be far apart to create large gaps
+    
+    const matchingRecords = [
+      { id: 1, url: 'http://olxi0fko.ukminlt.fun/iptv/HV7NY9RSQCHYZK/1066/index.m3u8', name: 'Дорама', logo: '', category: 'кино', nameTerms: ['кино'] },
+      { id: 50, url: 'http://olxi0fko.ukminlt.fun/iptv/HV7NY9RSQCHYZK/1066/index.m3u8', name: 'Дорама', logo: '', category: 'кино', nameTerms: ['кино'] },
+      { id: 100, url: 'http://olxi0fko.ukminlt.fun/iptv/HV7NY9RSQCHYZK/1066/index.m3u8', name: 'Дорама', logo: '', category: 'кино', nameTerms: ['кино'] },
+      { id: 150, url: 'http://olxi0fko.ukminlt.fun/iptv/HV7NY9RSQCHYZK/1066/index.m3u8', name: 'Дорама', logo: '', category: 'кино', nameTerms: ['кино'] },
+      { id: 200, url: 'http://olxi0fko.ukminlt.fun/iptv/HV7NY9RSQCHYZK/1066/index.m3u8', name: 'Дорама', logo: '', category: 'кино', nameTerms: ['кино'] }
+    ]
+
+    // Insert non-matching records between them to create gaps
+    for (let i = 1; i <= 200; i++) {
+      if (matchingRecords.find(r => r.id === i)) {
+        await db.insert(matchingRecords.find(r => r.id === i))
+      } else {
+        await db.insert({
+          id: i,
+          url: `http://example.com/${i}/index.m3u8`,
+          name: `Channel ${i}`,
+          logo: '',
+          category: 'other',
+          nameTerms: [`other${i}`]
+        })
+      }
+    }
+
+    await db.save()
+    await db.close()
+
+    // Reopen to force file-based reading
+    db = new Database(dbPath, {
+      fields: {
+        id: 'number',
+        url: 'string',
+        name: 'string',
+        logo: 'string',
+        category: 'string',
+        nameTerms: 'array:string'
+      },
+      indexes: {
+        nameTerms: 'array:string'
+      },
+      debugMode: false
+    })
+
+    await db.init()
+
+    // Query that returns only the 5 matching records (id: 1, 50, 100, 150, 200)
+    // These are non-adjacent and will be grouped together if they fit in 512KB
+    const query = { nameTerms: 'кино' }
+    const results = []
+    const errors = []
+
+    try {
+      for await (const record of db.walk(query)) {
+        // Try to access the record - this will fail if deserialization was incomplete
+        const recordStr = JSON.stringify(record)
+        
+        // Check if record is complete (has all expected fields)
+        if (!record.id || !record.url || !record.name || !record.category || !record.nameTerms) {
+          errors.push(`Incomplete record: ${recordStr.substring(0, 200)}`)
+        }
+        
+        // Verify UTF-8 characters are intact
+        if (record.name !== 'Дорама') {
+          errors.push(`UTF-8 corruption: expected 'Дорама', got '${record.name}'`)
+        }
+        
+        results.push(record)
+      }
+    } catch (error) {
+      // This is the bug we're looking for!
+      const errorMsg = error.message || ''
+      
+      if (errorMsg.includes('Failed to deserialize') || 
+          errorMsg.includes('Unexpected non-whitespace') ||
+          errorMsg.includes('JSON') ||
+          errorMsg.includes('position')) {
+        
+        // FAIL THE TEST - bug detected!
+        throw new Error(`🐛 BUG CONFIRMED: Deserialization error with non-adjacent ranges!\n\n` +
+          `Error: ${errorMsg}\n\n` +
+          `Root Cause: readGroupedRange() in FileHandler.mjs line 311 uses:\n` +
+          `  content.substring(relativeStart, relativeEnd)\n\n` +
+          `When multiple non-adjacent ranges are in the same buffer, substring() can extract:\n` +
+          `1. Truncated lines (ending mid-UTF-8 character)\n` +
+          `2. Multiple lines concatenated together\n` +
+          `3. Part of one line + part of another\n\n` +
+          `The fix must extract complete lines by finding newline boundaries.\n` +
+          `Errors collected: ${errors.join('; ')}`)
+      }
+      
+      // Re-throw other errors
+      throw error
+    }
+
+    // If we get here, verify no errors occurred
+    if (errors.length > 0) {
+      throw new Error(`Validation errors detected:\n${errors.join('\n')}`)
+    }
+
+    // Should have found 5 records
+    expect(results.length).toBe(5)
+    
+    // Verify all results are correct
+    results.forEach(result => {
+      expect(result.name).toBe('Дорама')
+      expect(result.category).toBe('кино')
+      expect(result.nameTerms).toContain('кино')
+    })
+
+    await db.destroy()
+  })
+})
+

package/test/index-line-number-regression.test.js

@@ -0,0 +1,100 @@
+import { Database } from '../src/Database.mjs'
+import fs from 'fs'
+
+describe('Term mapping index line number regression', () => {
+  let testDbPath
+  let testIdxPath
+
+  beforeEach(() => {
+    const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`
+    testDbPath = `test-line-number-regression-${uniqueSuffix}.jdb`
+    testIdxPath = testDbPath.replace('.jdb', '.idx.jdb')
+  })
+
+  afterEach(() => {
+    for (const filePath of [testDbPath, testIdxPath]) {
+      if (filePath && fs.existsSync(filePath)) {
+        try {
+          fs.unlinkSync(filePath)
+        } catch (error) {
+          console.warn(`⚠️  Failed to delete ${filePath}: ${error.message}`)
+        }
+      }
+    }
+  })
+
+  test('should index new term mapping values appended after reload', async () => {
+    // Seed database with a first record and persist it to disk
+    const initialDb = new Database(testDbPath, {
+      indexes: { nameTerms: 'array:string' },
+      debugMode: false
+    })
+    await initialDb.init()
+    await initialDb.insert({
+      id: '1',
+      name: 'CANAL DO CLIENTE HD',
+      nameTerms: ['cliente', 'canal']
+    })
+    await initialDb.save()
+    await initialDb.close()
+
+    // Reopen the same database and append a new record with different terms
+    const db = new Database(testDbPath, {
+      indexes: { nameTerms: 'array:string' },
+      debugMode: false
+    })
+    await db.init()
+
+    const firstQuery = await db.find({ nameTerms: 'cliente' })
+    expect(firstQuery.map(record => record.id)).toEqual(['1'])
+
+    await db.insert({
+      id: '2',
+      name: 'Telecine Fun HD',
+      nameTerms: ['telecine', 'fun']
+    })
+    await db.save()
+
+    const telecineResults = await db.find({ nameTerms: 'telecine' })
+    const clienteResults = await db.find({ nameTerms: 'cliente' })
+
+    expect(telecineResults.map(record => record.id)).toEqual(['2'])
+    expect(clienteResults.map(record => record.id)).toEqual(['1'])
+
+    await db.destroy()
+  })
+
+  test('should remove old term mapping entries when record is updated', async () => {
+    const db = new Database(testDbPath, {
+      indexes: { nameTerms: 'array:string' },
+      debugMode: false
+    })
+    await db.init()
+
+    await db.insert({
+      id: '1',
+      name: 'CANAL DO CLIENTE HD',
+      nameTerms: ['cliente', 'canal']
+    })
+    await db.save()
+
+    await db.update({ id: '1' }, {
+      name: 'Telecine Fun HD',
+      nameTerms: ['telecine', 'fun']
+    })
+
+    await db.save()
+
+    const telecineResults = await db.find({ nameTerms: 'telecine' })
+    const clienteResults = await db.find({ nameTerms: 'cliente' })
+    const combinedResults = await db.find({ nameTerms: { $all: ['telecine', 'fun'] } })
+
+    expect(telecineResults.map(record => record.id)).toEqual(['1'])
+    expect(clienteResults.map(record => record.id)).toEqual([])
+    expect(combinedResults.map(record => record.id)).toEqual(['1'])
+
+    await db.destroy()
+  })
+
+})
+

package/test/index-missing-index-data.test.js

@@ -0,0 +1,91 @@
+import { Database } from '../src/Database.mjs'
+import fs from 'fs'
+
+describe('Indexed query fallback when index data missing', () => {
+  let testDbPath
+  let testIdxPath
+
+  beforeEach(() => {
+    const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2)}`
+    testDbPath = `test-missing-index-${uniqueSuffix}.jdb`
+    testIdxPath = testDbPath.replace('.jdb', '.idx.jdb')
+  })
+
+  afterEach(() => {
+    for (const filePath of [testDbPath, testIdxPath]) {
+      if (filePath && fs.existsSync(filePath)) {
+        try {
+          fs.unlinkSync(filePath)
+        } catch (error) {
+          console.warn(`⚠️  Failed to delete ${filePath}: ${error.message}`)
+        }
+      }
+    }
+  })
+
+  test('should fall back to streaming when indexed field data is missing', async () => {
+    const db = new Database(testDbPath, {
+      indexes: {
+        channel: 'string',
+        start: 'number',
+        end: 'number'
+      },
+      debugMode: false
+    })
+    await db.init()
+
+    await db.insert({
+      id: '1',
+      channel: 'Sample Channel',
+      start: 1000,
+      end: 2000
+    })
+
+    await db.insert({
+      id: '2',
+      channel: 'Another Channel',
+      start: 3000,
+      end: 4000
+    })
+
+    await db.save()
+    await db.close()
+
+    const idxContentRaw = fs.readFileSync(testIdxPath, 'utf8')
+    const idxContent = JSON.parse(idxContentRaw)
+
+    if (!idxContent.index) {
+      idxContent.index = {}
+    }
+    if (!idxContent.index.data) {
+      idxContent.index.data = {}
+    }
+
+    // Simulate missing index data for channel field while keeping other index data intact
+    idxContent.index.data.channel = {}
+
+    fs.writeFileSync(testIdxPath, JSON.stringify(idxContent, null, 2), 'utf8')
+
+    const reopenedDb = new Database(testDbPath, {
+      create: false,
+      indexes: {
+        channel: 'string',
+        start: 'number',
+        end: 'number'
+      },
+      allowIndexRebuild: true, // Enable rebuild for partial index corruption
+      debugMode: false
+    })
+
+    await reopenedDb.init()
+
+    const count = await reopenedDb.count({ channel: 'Sample Channel' })
+    expect(count).toBe(1)
+
+    const results = await reopenedDb.find({ channel: 'Sample Channel' })
+    expect(results.map(record => record.id)).toEqual(['1'])
+
+    await reopenedDb.destroy()
+  })
+})
+