odac 1.4.6 → 1.4.8

package/test/Database/WriteBuffer/_recoverFromCheckpoint.test.js

@@ -0,0 +1,207 @@
+'use strict'
+
+const cluster = require('node:cluster')
+
+/**
+ * Tests WriteBuffer LMDB checkpoint and crash recovery.
+ * Why: Validates zero data loss guarantee — buffered data survives process crashes
+ * via periodic LMDB checkpoints and is recovered on next startup.
+ */
+
+let knexLib, db, storageData
+
+function createMockStorage() {
+  storageData = new Map()
+  return {
+    isReady: () => true,
+    put: (key, value) => storageData.set(key, value),
+    remove: key => storageData.delete(key),
+    get: key => storageData.get(key) ?? null,
+    getRange: ({start, end}) => {
+      const results = []
+      for (const [key, value] of storageData) {
+        if (key >= start && key < end) {
+          results.push({key, value})
+        }
+      }
+      return results
+    }
+  }
+}
+
+beforeEach(async () => {
+  jest.resetModules()
+
+  knexLib = require('knex')
+  db = knexLib({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true})
+
+  await db.schema.createTable('posts', table => {
+    table.integer('id').primary()
+    table.integer('views').defaultTo(0)
+  })
+  await db('posts').insert({id: 1, views: 100})
+
+  Object.defineProperty(cluster, 'isPrimary', {value: true, configurable: true})
+})
+
+afterEach(async () => {
+  await db.destroy()
+  delete global.Odac
+})
+
+describe('WriteBuffer - Checkpoint', () => {
+  it('should write counter deltas to LMDB on checkpoint', async () => {
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: createMockStorage(),
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 5)
+    await WriteBuffer._writeCheckpoint()
+
+    const checkpoint = storageData.get('wb:c:default:posts:1:views')
+    expect(checkpoint).toBeDefined()
+    expect(checkpoint.delta).toBe(5)
+    expect(checkpoint.base).toBe(100)
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+
+  it('should write queue rows to LMDB on checkpoint', async () => {
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: createMockStorage(),
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+
+    await WriteBuffer.insert('default', 'activity_log', {user_id: 1, action: 'view'})
+    await WriteBuffer.insert('default', 'activity_log', {user_id: 2, action: 'click'})
+    await WriteBuffer._writeCheckpoint()
+
+    const checkpoint = storageData.get('wb:q:default:activity_log')
+    expect(checkpoint).toBeDefined()
+    expect(checkpoint).toHaveLength(2)
+    expect(checkpoint[0].action).toBe('view')
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+})
+
+describe('WriteBuffer - Recovery', () => {
+  it('should recover counter deltas from LMDB on startup', async () => {
+    // Simulate crash: write checkpoint data before init
+    const mockStorage = createMockStorage()
+    storageData.set('wb:c:default:posts:1:views', {delta: 7, base: 100})
+
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: mockStorage,
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+
+    // Should recover the delta from checkpoint
+    const result = await WriteBuffer.get('default', 'posts', 1, 'views')
+    expect(result).toBe(107) // base 100 + recovered delta 7
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+
+  it('should recover queue rows from LMDB on startup', async () => {
+    await db.schema.createTable('activity_log', table => {
+      table.increments('id')
+      table.integer('user_id')
+      table.string('action', 50)
+    })
+
+    const mockStorage = createMockStorage()
+    storageData.set('wb:q:default:activity_log', [{user_id: 1, action: 'recovered_view'}])
+
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: mockStorage,
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+
+    // Flush recovered data
+    await WriteBuffer.flush()
+
+    const rows = await db('activity_log').select()
+    expect(rows).toHaveLength(1)
+    expect(rows[0].action).toBe('recovered_view')
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+
+  it('should merge recovered data with new increments', async () => {
+    const mockStorage = createMockStorage()
+    storageData.set('wb:c:default:posts:1:views', {delta: 5, base: 100})
+
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: mockStorage,
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+
+    // Add more increments on top of recovered data
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 3)
+
+    const result = await WriteBuffer.get('default', 'posts', 1, 'views')
+    expect(result).toBe(108) // base 100 + recovered 5 + new 3
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+
+  it('should clear LMDB checkpoint after successful flush', async () => {
+    const mockStorage = createMockStorage()
+    storageData.set('wb:c:default:posts:1:views', {delta: 5, base: 100})
+
+    const Ipc = require('../../../src/Ipc')
+    global.Odac = {
+      Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+      Storage: mockStorage,
+      Ipc
+    }
+    await Ipc.init()
+
+    const WriteBuffer = require('../../../src/Database/WriteBuffer')
+    await WriteBuffer.init({default: db})
+    await WriteBuffer.flush()
+
+    // Checkpoint data should be cleared
+    expect(storageData.has('wb:c:default:posts:1:views')).toBe(false)
+
+    await WriteBuffer.close()
+    await Ipc.close()
+  })
+})

package/test/Database/WriteBuffer/buffer.test.js

@@ -0,0 +1,143 @@
+'use strict'
+
+const cluster = require('node:cluster')
+
+/**
+ * Tests the chainable buffer API exposed via Database.js proxy.
+ * Why: Validates that the Odac.DB.table.buffer.where(id).update(data) pattern
+ * correctly delegates to WriteBuffer's internal methods.
+ */
+
+let knexLib, db
+
+beforeEach(async () => {
+  jest.resetModules()
+
+  knexLib = require('knex')
+  db = knexLib({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true})
+
+  await db.schema.createTable('posts', table => {
+    table.integer('id').primary()
+    table.integer('views').defaultTo(0)
+    table.string('title', 255)
+  })
+
+  await db.schema.createTable('activity_log', table => {
+    table.increments('id')
+    table.integer('user_id')
+    table.string('action', 50)
+  })
+
+  await db('posts').insert([
+    {id: 1, views: 100, title: 'First Post'},
+    {id: 2, views: 200, title: 'Second Post'}
+  ])
+
+  Object.defineProperty(cluster, 'isPrimary', {value: true, configurable: true})
+
+  const Ipc = require('../../../src/Ipc')
+  global.Odac = {
+    Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+    Storage: {
+      isReady: () => false,
+      put: jest.fn(),
+      remove: jest.fn(),
+      getRange: () => []
+    },
+    Ipc
+  }
+  await Ipc.init()
+
+  // Initialize WriteBuffer with our test DB
+  const writeBuffer = require('../../../src/Database/WriteBuffer')
+  await writeBuffer.init({default: db})
+
+  // Wire up Database.js proxy
+  const DB = require('../../../src/Database')
+  DB.connections = {default: db}
+})
+
+afterEach(async () => {
+  const writeBuffer = require('../../../src/Database/WriteBuffer')
+  await writeBuffer.close()
+  await Odac.Ipc.close()
+  await db.destroy()
+  delete global.Odac
+})
+
+describe('Database.js Proxy - buffer.where().update()', () => {
+  it('should buffer and flush via Odac.DB.posts.buffer.where(id).update()', async () => {
+    const DB = require('../../../src/Database')
+
+    await DB.posts.buffer.where(1).update({title: 'Updated Title'})
+    await DB.posts.buffer.flush()
+
+    const row = await db('posts').where({id: 1}).first()
+    expect(row.title).toBe('Updated Title')
+  })
+
+  it('should merge multiple updates via chainable API', async () => {
+    const DB = require('../../../src/Database')
+
+    await DB.posts.buffer.where(1).update({title: 'New Title'})
+    await DB.posts.buffer.where(1).update({title: 'Final Title'})
+    await DB.posts.buffer.flush()
+
+    const row = await db('posts').where({id: 1}).first()
+    expect(row.title).toBe('Final Title')
+  })
+})
+
+describe('Database.js Proxy - buffer.where().increment()', () => {
+  it('should increment via Odac.DB.posts.buffer.where(id).increment(col)', async () => {
+    const DB = require('../../../src/Database')
+
+    const result = await DB.posts.buffer.where(1).increment('views')
+    expect(result).toBe(101)
+  })
+
+  it('should support custom delta', async () => {
+    const DB = require('../../../src/Database')
+
+    const result = await DB.posts.buffer.where(1).increment('views', 5)
+    expect(result).toBe(105)
+  })
+})
+
+describe('Database.js Proxy - buffer.where().get()', () => {
+  it('should get buffered value via Odac.DB.posts.buffer.where(id).get(col)', async () => {
+    const DB = require('../../../src/Database')
+
+    await DB.posts.buffer.where(1).increment('views', 10)
+    const result = await DB.posts.buffer.where(1).get('views')
+    expect(result).toBe(110)
+  })
+})
+
+describe('Database.js Proxy - buffer.insert()', () => {
+  it('should buffer insert via Odac.DB.activity_log.buffer.insert(row)', async () => {
+    const DB = require('../../../src/Database')
+
+    await DB.activity_log.buffer.insert({user_id: 1, action: 'view'})
+    await DB.activity_log.buffer.insert({user_id: 2, action: 'click'})
+    await DB.activity_log.buffer.flush()
+
+    const rows = await db('activity_log').select()
+    expect(rows).toHaveLength(2)
+  })
+})
+
+describe('Database.js Proxy - buffer.flush()', () => {
+  it('should flush all buffered data for the table', async () => {
+    const DB = require('../../../src/Database')
+
+    await DB.posts.buffer.where(1).increment('views', 5)
+    await DB.posts.buffer.where(2).update({title: 'Changed'})
+    await DB.posts.buffer.flush()
+
+    const row1 = await db('posts').where({id: 1}).first()
+    const row2 = await db('posts').where({id: 2}).first()
+    expect(row1.views).toBe(105)
+    expect(row2.title).toBe('Changed')
+  })
+})

package/test/Database/WriteBuffer/flush.test.js

@@ -0,0 +1,192 @@
+'use strict'
+
+const cluster = require('node:cluster')
+
+/**
+ * Tests WriteBuffer flush operations: counter flush (batch UPDATE) and queue flush (batch INSERT).
+ * Why: Validates that buffered data is correctly persisted to the database,
+ * with proper transaction handling and error recovery.
+ */
+
+let knexLib, db, WriteBuffer
+
+beforeEach(async () => {
+  jest.resetModules()
+
+  knexLib = require('knex')
+  db = knexLib({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true})
+
+  await db.schema.createTable('posts', table => {
+    table.integer('id').primary()
+    table.integer('views').defaultTo(0)
+    table.integer('likes').defaultTo(0)
+    table.string('title', 255)
+  })
+
+  await db('posts').insert([
+    {id: 1, views: 100, likes: 10, title: 'First Post'},
+    {id: 2, views: 200, likes: 20, title: 'Second Post'}
+  ])
+
+  Object.defineProperty(cluster, 'isPrimary', {value: true, configurable: true})
+
+  const Ipc = require('../../../src/Ipc')
+  global.Odac = {
+    Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+    Storage: {
+      isReady: () => false,
+      put: jest.fn(),
+      remove: jest.fn(),
+      getRange: () => []
+    },
+    Ipc
+  }
+  await Ipc.init()
+
+  WriteBuffer = require('../../../src/Database/WriteBuffer')
+  await WriteBuffer.init({default: db})
+})
+
+afterEach(async () => {
+  await WriteBuffer.close()
+  await Odac.Ipc.close()
+  await db.destroy()
+  delete global.Odac
+})
+
+describe('WriteBuffer - Counter Flush', () => {
+  it('should persist accumulated deltas to the database', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 5)
+    await WriteBuffer.increment('default', 'posts', 2, 'views', 10)
+
+    await WriteBuffer.flush()
+
+    const row1 = await db('posts').where({id: 1}).first()
+    const row2 = await db('posts').where({id: 2}).first()
+    expect(row1.views).toBe(105)
+    expect(row2.views).toBe(210)
+  })
+
+  it('should flush multiple columns for the same row', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 3)
+    await WriteBuffer.increment('default', 'posts', 1, 'likes', 7)
+
+    await WriteBuffer.flush()
+
+    const row = await db('posts').where({id: 1}).first()
+    expect(row.views).toBe(103)
+    expect(row.likes).toBe(17)
+  })
+
+  it('should clear counter index after successful flush', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 5)
+    await WriteBuffer.flush()
+
+    // Counter index should be empty after flush
+    const remaining = await Odac.Ipc.smembers('wb:idx:counters')
+    expect(remaining).toHaveLength(0)
+  })
+
+  it('should update base after flush so subsequent reads are correct', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 5)
+    await WriteBuffer.flush()
+
+    // New increment after flush
+    const result = await WriteBuffer.increment('default', 'posts', 1, 'views', 2)
+    expect(result).toBe(107) // base updated to 105, + 2
+
+    await WriteBuffer.flush()
+    const row = await db('posts').where({id: 1}).first()
+    expect(row.views).toBe(107)
+  })
+
+  it('should accumulate new deltas during flush correctly', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 10)
+    await WriteBuffer.flush()
+
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 3)
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 2)
+    await WriteBuffer.flush()
+
+    const row = await db('posts').where({id: 1}).first()
+    expect(row.views).toBe(115) // 100 + 10 + 5
+  })
+
+  it('should scope flush to specific table when provided', async () => {
+    await db.schema.createTable('comments', table => {
+      table.integer('id').primary()
+      table.integer('votes').defaultTo(0)
+    })
+    await db('comments').insert({id: 1, votes: 50})
+
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 5)
+    await WriteBuffer.increment('default', 'comments', 1, 'votes', 3)
+
+    // Flush only posts
+    await WriteBuffer.flush('default', 'posts')
+
+    const post = await db('posts').where({id: 1}).first()
+    const comment = await db('comments').where({id: 1}).first()
+    expect(post.views).toBe(105) // Flushed
+    expect(comment.votes).toBe(50) // Not flushed
+  })
+})
+
+describe('WriteBuffer - Queue Flush (Batch Insert)', () => {
+  it('should batch insert queued rows', async () => {
+    await db.schema.createTable('activity_log', table => {
+      table.increments('id')
+      table.integer('user_id')
+      table.string('action', 50)
+    })
+
+    await WriteBuffer.insert('default', 'activity_log', {user_id: 1, action: 'view'})
+    await WriteBuffer.insert('default', 'activity_log', {user_id: 2, action: 'click'})
+    await WriteBuffer.insert('default', 'activity_log', {user_id: 1, action: 'scroll'})
+
+    await WriteBuffer.flush()
+
+    const rows = await db('activity_log').select()
+    expect(rows).toHaveLength(3)
+    expect(rows[0].action).toBe('view')
+    expect(rows[1].action).toBe('click')
+    expect(rows[2].action).toBe('scroll')
+  })
+
+  it('should clear queue after successful flush', async () => {
+    await db.schema.createTable('events', table => {
+      table.increments('id')
+      table.string('type', 50)
+    })
+
+    await WriteBuffer.insert('default', 'events', {type: 'pageview'})
+    await WriteBuffer.flush()
+
+    const queue = await Odac.Ipc.lrange('wb:q:default:events', 0, -1)
+    expect(queue).toHaveLength(0)
+  })
+
+  it('should handle empty queues gracefully', async () => {
+    await expect(WriteBuffer.flush()).resolves.not.toThrow()
+  })
+
+  it('should auto-flush when maxQueueSize is reached', async () => {
+    await db.schema.createTable('logs', table => {
+      table.increments('id')
+      table.string('msg', 50)
+    })
+
+    // Set low threshold for testing
+    WriteBuffer._config.maxQueueSize = 3
+
+    await WriteBuffer.insert('default', 'logs', {msg: 'a'})
+    await WriteBuffer.insert('default', 'logs', {msg: 'b'})
+    await WriteBuffer.insert('default', 'logs', {msg: 'c'}) // Triggers auto-flush
+
+    // Wait a tick for async auto-flush
+    await new Promise(r => setTimeout(r, 50))
+
+    const rows = await db('logs').select()
+    expect(rows.length).toBeGreaterThanOrEqual(3)
+  })
+})

package/test/Database/WriteBuffer/get.test.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const cluster = require('node:cluster')
+
+/**
+ * Tests WriteBuffer.get().
+ * Why: Validates that get() returns the accurate current value (DB base + buffered delta)
+ * without flushing to the database.
+ */
+
+let knexLib, db, WriteBuffer
+
+beforeEach(async () => {
+  jest.resetModules()
+
+  knexLib = require('knex')
+  db = knexLib({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true})
+
+  await db.schema.createTable('posts', table => {
+    table.integer('id').primary()
+    table.integer('views').defaultTo(0)
+    table.string('title', 255)
+  })
+
+  await db('posts').insert([
+    {id: 1, views: 100, title: 'First Post'},
+    {id: 2, views: 200, title: 'Second Post'}
+  ])
+
+  Object.defineProperty(cluster, 'isPrimary', {value: true, configurable: true})
+
+  const Ipc = require('../../../src/Ipc')
+  global.Odac = {
+    Config: {buffer: {flushInterval: 999999, checkpointInterval: 999999}},
+    Storage: {
+      isReady: () => false,
+      put: jest.fn(),
+      remove: jest.fn(),
+      getRange: () => []
+    },
+    Ipc
+  }
+  await Ipc.init()
+
+  WriteBuffer = require('../../../src/Database/WriteBuffer')
+  await WriteBuffer.init({default: db})
+})
+
+afterEach(async () => {
+  await WriteBuffer.close()
+  await Odac.Ipc.close()
+  await db.destroy()
+  delete global.Odac
+})
+
+describe('WriteBuffer - get()', () => {
+  it('should return base value from DB when no buffer exists', async () => {
+    const result = await WriteBuffer.get('default', 'posts', 1, 'views')
+    expect(result).toBe(100)
+  })
+
+  it('should return base + delta when buffer exists', async () => {
+    await WriteBuffer.increment('default', 'posts', 1, 'views', 7)
+    const result = await WriteBuffer.get('default', 'posts', 1, 'views')
+    expect(result).toBe(107)
+  })
+
+  it('should return 0 for non-existent rows', async () => {
+    const result = await WriteBuffer.get('default', 'posts', 999, 'views')
+    expect(result).toBe(0)
+  })
+})