@dotdo/postgres 0.1.0

package/src/worker/wal.test.ts

@@ -0,0 +1,618 @@
+/**
+ * WAL (Write-Ahead Log) Capture Tests
+ * Task: postgres-ctf - WAL capture mechanism in PostgresDO
+ *
+ * Tests for WAL capture, buffering, and storage functionality.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import {
+  WALManager,
+  DOWALStorage,
+  parseWriteOperation,
+  serializeWALEntry,
+  deserializeWALEntry,
+  type WALEntry,
+  type WALConfig,
+  type WALStorage,
+  type SerializedWALEntry,
+} from './wal'
+
+// Mock DurableObjectState storage
+function createMockStorage() {
+  const store = new Map<string, unknown>()
+
+  return {
+    get: vi.fn(async <T>(key: string): Promise<T | undefined> => {
+      return store.get(key) as T | undefined
+    }),
+    put: vi.fn(async (entries: Record<string, unknown>): Promise<void> => {
+      for (const [key, value] of Object.entries(entries)) {
+        store.set(key, value)
+      }
+    }),
+    list: vi.fn(async <T>(options: { start?: string; end?: string; limit?: number }): Promise<Map<string, T>> => {
+      const result = new Map<string, T>()
+      const entries = Array.from(store.entries())
+        .filter(([key]) => {
+          if (options.start && key < options.start) return false
+          if (options.end && key >= options.end) return false
+          return true
+        })
+        .sort(([a], [b]) => a.localeCompare(b))
+        .slice(0, options.limit || 100)
+
+      for (const [key, value] of entries) {
+        result.set(key, value as T)
+      }
+      return result
+    }),
+    delete: vi.fn(),
+    _store: store, // Expose for testing
+  }
+}
+
+function createMockState() {
+  return {
+    storage: createMockStorage(),
+    id: { toString: () => 'test-do-id' },
+    blockConcurrencyWhile: vi.fn((fn) => fn()),
+    waitUntil: vi.fn(),
+  } as unknown as DurableObjectState
+}
+
+describe('WAL Module', () => {
+  describe('parseWriteOperation', () => {
+    it('should parse INSERT statements', () => {
+      const result = parseWriteOperation('INSERT INTO users (id, name) VALUES (1, \'Alice\')')
+      expect(result).toEqual({
+        operation: 'INSERT',
+        schema: 'public',
+        table: 'users',
+      })
+    })
+
+    it('should parse INSERT with schema', () => {
+      const result = parseWriteOperation('INSERT INTO myschema.users (id) VALUES (1)')
+      expect(result).toEqual({
+        operation: 'INSERT',
+        schema: 'myschema',
+        table: 'users',
+      })
+    })
+
+    it('should parse INSERT with quoted table name', () => {
+      const result = parseWriteOperation('INSERT INTO "UserTable" (id) VALUES (1)')
+      expect(result).toEqual({
+        operation: 'INSERT',
+        schema: 'public',
+        table: 'UserTable',
+      })
+    })
+
+    it('should parse UPDATE statements', () => {
+      const result = parseWriteOperation('UPDATE users SET name = \'Bob\' WHERE id = 1')
+      expect(result).toEqual({
+        operation: 'UPDATE',
+        schema: 'public',
+        table: 'users',
+      })
+    })
+
+    it('should parse UPDATE with schema', () => {
+      const result = parseWriteOperation('UPDATE myschema.users SET name = \'Bob\'')
+      expect(result).toEqual({
+        operation: 'UPDATE',
+        schema: 'myschema',
+        table: 'users',
+      })
+    })
+
+    it('should parse DELETE statements', () => {
+      const result = parseWriteOperation('DELETE FROM users WHERE id = 1')
+      expect(result).toEqual({
+        operation: 'DELETE',
+        schema: 'public',
+        table: 'users',
+      })
+    })
+
+    it('should parse DELETE with schema', () => {
+      const result = parseWriteOperation('DELETE FROM myschema.users WHERE id = 1')
+      expect(result).toEqual({
+        operation: 'DELETE',
+        schema: 'myschema',
+        table: 'users',
+      })
+    })
+
+    it('should return null for SELECT statements', () => {
+      const result = parseWriteOperation('SELECT * FROM users')
+      expect(result).toBeNull()
+    })
+
+    it('should return null for CREATE TABLE', () => {
+      const result = parseWriteOperation('CREATE TABLE users (id INT)')
+      expect(result).toBeNull()
+    })
+
+    it('should handle case insensitivity', () => {
+      const result = parseWriteOperation('insert into Users (id) values (1)')
+      expect(result).toEqual({
+        operation: 'INSERT',
+        schema: 'public',
+        table: 'users',
+      })
+    })
+
+    it('should handle extra whitespace', () => {
+      const result = parseWriteOperation('  INSERT   INTO   users   (id)   VALUES   (1)  ')
+      expect(result).toEqual({
+        operation: 'INSERT',
+        schema: 'public',
+        table: 'users',
+      })
+    })
+  })
+
+  describe('serializeWALEntry / deserializeWALEntry', () => {
+    it('should serialize and deserialize WAL entry', () => {
+      const entry: WALEntry = {
+        lsn: 12345n,
+        operation: 'INSERT',
+        schema: 'public',
+        table: 'users',
+        timestamp: Date.now(),
+        newRow: { id: 1, name: 'Alice' },
+      }
+
+      const serialized = serializeWALEntry(entry)
+      expect(serialized.lsn).toBe('12345')
+      expect(typeof serialized.lsn).toBe('string')
+
+      const deserialized = deserializeWALEntry(serialized)
+      expect(deserialized.lsn).toBe(12345n)
+      expect(deserialized.operation).toBe('INSERT')
+      expect(deserialized.newRow).toEqual({ id: 1, name: 'Alice' })
+    })
+
+    it('should handle large LSN values', () => {
+      const entry: WALEntry = {
+        lsn: 9007199254740991n, // Max safe integer + 1
+        operation: 'UPDATE',
+        schema: 'public',
+        table: 'users',
+        timestamp: Date.now(),
+      }
+
+      const serialized = serializeWALEntry(entry)
+      const deserialized = deserializeWALEntry(serialized)
+      expect(deserialized.lsn).toBe(9007199254740991n)
+    })
+  })
+
+  describe('WALManager', () => {
+    let walManager: WALManager
+
+    beforeEach(() => {
+      walManager = new WALManager()
+    })
+
+    describe('configuration', () => {
+      it('should have default configuration', () => {
+        const config = walManager.getConfig()
+        expect(config.enabled).toBe(true)
+        expect(config.captureMode).toBe('full')
+        expect(config.maxBufferEntries).toBe(1000)
+        expect(config.maxBufferBytes).toBe(1024 * 1024)
+      })
+
+      it('should allow custom configuration', () => {
+        const customManager = new WALManager({
+          enabled: false,
+          captureMode: 'delta',
+          maxBufferEntries: 500,
+        })
+
+        const config = customManager.getConfig()
+        expect(config.enabled).toBe(false)
+        expect(config.captureMode).toBe('delta')
+        expect(config.maxBufferEntries).toBe(500)
+      })
+
+      it('should update configuration', () => {
+        walManager.setConfig({ maxBufferEntries: 2000 })
+        expect(walManager.getConfig().maxBufferEntries).toBe(2000)
+      })
+
+      it('should enable/disable capture', () => {
+        expect(walManager.isEnabled()).toBe(true)
+        walManager.setEnabled(false)
+        expect(walManager.isEnabled()).toBe(false)
+        walManager.setEnabled(true)
+        expect(walManager.isEnabled()).toBe(true)
+      })
+    })
+
+    describe('capture', () => {
+      it('should capture INSERT operations', () => {
+        const entry = walManager.capture('INSERT INTO users (id, name) VALUES (1, \'Alice\')', {
+          rows: [{ id: 1, name: 'Alice' }],
+        })
+
+        expect(entry).not.toBeNull()
+        expect(entry!.operation).toBe('INSERT')
+        expect(entry!.table).toBe('users')
+        expect(entry!.newRow).toEqual({ id: 1, name: 'Alice' })
+        expect(entry!.lsn).toBe(1n)
+      })
+
+      it('should capture UPDATE operations', () => {
+        const entry = walManager.capture('UPDATE users SET name = \'Bob\' WHERE id = 1', {
+          rows: [{ id: 1, name: 'Bob' }],
+          oldRows: [{ id: 1, name: 'Alice' }],
+        })
+
+        expect(entry).not.toBeNull()
+        expect(entry!.operation).toBe('UPDATE')
+        expect(entry!.newRow).toEqual({ id: 1, name: 'Bob' })
+        expect(entry!.oldRow).toEqual({ id: 1, name: 'Alice' })
+      })
+
+      it('should capture DELETE operations', () => {
+        const entry = walManager.capture('DELETE FROM users WHERE id = 1', {
+          oldRows: [{ id: 1, name: 'Alice' }],
+        })
+
+        expect(entry).not.toBeNull()
+        expect(entry!.operation).toBe('DELETE')
+        expect(entry!.oldRow).toEqual({ id: 1, name: 'Alice' })
+      })
+
+      it('should not capture SELECT operations', () => {
+        const entry = walManager.capture('SELECT * FROM users', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(entry).toBeNull()
+      })
+
+      it('should not capture when disabled', () => {
+        walManager.setEnabled(false)
+        const entry = walManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(entry).toBeNull()
+      })
+
+      it('should increment LSN for each capture', () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (3)', { rows: [{ id: 3 }] })
+
+        expect(walManager.getCurrentLsn()).toBe(3n)
+      })
+
+      it('should include timestamp', () => {
+        const before = Date.now()
+        const entry = walManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+        const after = Date.now()
+
+        expect(entry!.timestamp).toBeGreaterThanOrEqual(before)
+        expect(entry!.timestamp).toBeLessThanOrEqual(after)
+      })
+
+      it('should include SQL when configured', () => {
+        const sqlManager = new WALManager({ includeSQL: true })
+        const entry = sqlManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(entry!.sql).toBe('INSERT INTO users (id) VALUES (1)')
+      })
+
+      it('should not include SQL by default', () => {
+        const entry = walManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(entry!.sql).toBeUndefined()
+      })
+    })
+
+    describe('table filtering', () => {
+      it('should filter by includeTables', () => {
+        const filteredManager = new WALManager({
+          includeTables: ['allowed_table'],
+        })
+
+        const allowed = filteredManager.capture('INSERT INTO allowed_table (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+        const excluded = filteredManager.capture('INSERT INTO other_table (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(allowed).not.toBeNull()
+        expect(excluded).toBeNull()
+      })
+
+      it('should filter by excludeTables', () => {
+        const filteredManager = new WALManager({
+          excludeTables: ['excluded_table'],
+        })
+
+        const allowed = filteredManager.capture('INSERT INTO allowed_table (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+        const excluded = filteredManager.capture('INSERT INTO excluded_table (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(allowed).not.toBeNull()
+        expect(excluded).toBeNull()
+      })
+    })
+
+    describe('operation filtering', () => {
+      it('should filter by operations', () => {
+        const insertOnlyManager = new WALManager({
+          operations: ['INSERT'],
+        })
+
+        const insert = insertOnlyManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+        const update = insertOnlyManager.capture('UPDATE users SET id = 2', {
+          rows: [{ id: 2 }],
+        })
+        const del = insertOnlyManager.capture('DELETE FROM users WHERE id = 1', {
+          oldRows: [{ id: 1 }],
+        })
+
+        expect(insert).not.toBeNull()
+        expect(update).toBeNull()
+        expect(del).toBeNull()
+      })
+    })
+
+    describe('buffer management', () => {
+      it('should buffer entries', () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        const buffer = walManager.getBufferedEntries()
+        expect(buffer.length).toBe(2)
+      })
+
+      it('should track buffer statistics', () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        const stats = walManager.getStats()
+        expect(stats.entryCount).toBe(2)
+        expect(stats.memoryBytes).toBeGreaterThan(0)
+        expect(stats.oldestLsn).toBe(1n)
+        expect(stats.newestLsn).toBe(2n)
+        expect(stats.totalEntriesCaptured).toBe(2n)
+      })
+
+      it('should drop oldest entries when buffer is full (no storage)', () => {
+        const smallBuffer = new WALManager({
+          maxBufferEntries: 3,
+        })
+
+        smallBuffer.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        smallBuffer.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+        smallBuffer.capture('INSERT INTO users (id) VALUES (3)', { rows: [{ id: 3 }] })
+        smallBuffer.capture('INSERT INTO users (id) VALUES (4)', { rows: [{ id: 4 }] })
+
+        const stats = smallBuffer.getStats()
+        expect(stats.entryCount).toBe(3)
+        expect(stats.oldestLsn).toBe(2n)
+        expect(stats.entriesDropped).toBe(1n)
+      })
+
+      it('should clear buffer', () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        const cleared = walManager.clearBuffer()
+        expect(cleared).toBe(2)
+        expect(walManager.getStats().entryCount).toBe(0)
+        expect(walManager.getStats().entriesDropped).toBe(2n)
+      })
+    })
+
+    describe('event listeners', () => {
+      it('should emit entry events', () => {
+        const entries: WALEntry[] = []
+        walManager.onEntry((entry) => entries.push(entry))
+
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        expect(entries.length).toBe(2)
+        expect(entries[0].lsn).toBe(1n)
+        expect(entries[1].lsn).toBe(2n)
+      })
+
+      it('should unsubscribe from entry events', () => {
+        const entries: WALEntry[] = []
+        const unsubscribe = walManager.onEntry((entry) => entries.push(entry))
+
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        unsubscribe()
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        expect(entries.length).toBe(1)
+      })
+    })
+
+    describe('flush', () => {
+      it('should flush buffer and return result', async () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+        const result = await walManager.flush()
+        expect(result.entriesFlushed).toBe(2)
+        expect(result.lsnRange).toEqual({ start: 1n, end: 2n })
+        expect(result.durationMs).toBeGreaterThanOrEqual(0)
+      })
+
+      it('should clear buffer after flush', async () => {
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        await walManager.flush()
+
+        expect(walManager.getStats().entryCount).toBe(0)
+        expect(walManager.getStats().entriesFlushed).toBe(1n)
+      })
+
+      it('should emit flush events', async () => {
+        const flushResults: { entriesFlushed: number }[] = []
+        walManager.onFlush((result) => flushResults.push({ entriesFlushed: result.entriesFlushed }))
+
+        walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+        await walManager.flush()
+
+        expect(flushResults.length).toBe(1)
+        expect(flushResults[0].entriesFlushed).toBe(1)
+      })
+
+      it('should handle empty buffer flush', async () => {
+        const result = await walManager.flush()
+        expect(result.entriesFlushed).toBe(0)
+        expect(result.lsnRange).toBeNull()
+      })
+    })
+
+    describe('transaction context', () => {
+      it('should include transaction ID when set', () => {
+        walManager.beginTransaction('tx-123')
+        const entry = walManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+        walManager.endTransaction()
+
+        expect(entry!.transactionId).toBe('tx-123')
+      })
+
+      it('should not include transaction ID after end', () => {
+        walManager.beginTransaction('tx-123')
+        walManager.endTransaction()
+        const entry = walManager.capture('INSERT INTO users (id) VALUES (1)', {
+          rows: [{ id: 1 }],
+        })
+
+        expect(entry!.transactionId).toBeUndefined()
+      })
+    })
+
+    describe('delta mode', () => {
+      it('should calculate changed columns in delta mode', () => {
+        const deltaManager = new WALManager({ captureMode: 'delta' })
+
+        const entry = deltaManager.capture('UPDATE users SET name = \'Bob\', age = 30 WHERE id = 1', {
+          rows: [{ id: 1, name: 'Bob', age: 30, email: 'bob@test.com' }],
+          oldRows: [{ id: 1, name: 'Alice', age: 30, email: 'bob@test.com' }],
+        })
+
+        expect(entry!.changedColumns).toEqual(['name'])
+      })
+    })
+  })
+
+  describe('DOWALStorage', () => {
+    let storage: DOWALStorage
+    let mockState: DurableObjectState
+
+    beforeEach(() => {
+      mockState = createMockState()
+      storage = new DOWALStorage(mockState)
+    })
+
+    it('should store WAL entries', async () => {
+      const entries: SerializedWALEntry[] = [
+        { lsn: '1', operation: 'INSERT', schema: 'public', table: 'users', timestamp: Date.now() },
+        { lsn: '2', operation: 'UPDATE', schema: 'public', table: 'users', timestamp: Date.now() },
+      ]
+
+      await storage.store(entries)
+
+      expect(mockState.storage.put).toHaveBeenCalled()
+    })
+
+    it('should get last LSN', async () => {
+      // First store some entries
+      await storage.store([
+        { lsn: '100', operation: 'INSERT', schema: 'public', table: 'users', timestamp: Date.now() },
+      ])
+
+      const lastLsn = await storage.getLastLsn()
+      expect(lastLsn).toBe(100n)
+    })
+
+    it('should return null for empty storage', async () => {
+      const lastLsn = await storage.getLastLsn()
+      expect(lastLsn).toBeNull()
+    })
+
+    it('should read entries from storage', async () => {
+      // Store some entries
+      await storage.store([
+        { lsn: '1', operation: 'INSERT', schema: 'public', table: 'users', timestamp: Date.now() },
+        { lsn: '2', operation: 'UPDATE', schema: 'public', table: 'users', timestamp: Date.now() },
+        { lsn: '3', operation: 'DELETE', schema: 'public', table: 'users', timestamp: Date.now() },
+      ])
+
+      const entries = await storage.readEntries(1n)
+      expect(entries.length).toBeGreaterThanOrEqual(1)
+    })
+  })
+
+  describe('WALManager with Storage', () => {
+    let walManager: WALManager
+    let mockState: DurableObjectState
+    let storage: DOWALStorage
+
+    beforeEach(() => {
+      mockState = createMockState()
+      storage = new DOWALStorage(mockState)
+      walManager = new WALManager({}, storage)
+    })
+
+    it('should initialize from storage', async () => {
+      // Pre-populate storage with lastLsn
+      await storage.store([
+        { lsn: '100', operation: 'INSERT', schema: 'public', table: 'users', timestamp: Date.now() },
+      ])
+
+      await walManager.initialize()
+
+      // New entries should start from 101
+      walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+      expect(walManager.getCurrentLsn()).toBe(101n)
+    })
+
+    it('should flush to storage', async () => {
+      walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+      walManager.capture('INSERT INTO users (id) VALUES (2)', { rows: [{ id: 2 }] })
+
+      await walManager.flush()
+
+      const entries = await storage.readEntries(0n)
+      expect(entries.length).toBe(2)
+    })
+
+    it('should read from storage after flush', async () => {
+      walManager.capture('INSERT INTO users (id) VALUES (1)', { rows: [{ id: 1 }] })
+      await walManager.flush()
+
+      const entries = await walManager.readFromStorage(0n)
+      expect(entries.length).toBe(1)
+      expect(entries[0].lsn).toBe(1n)
+    })
+  })
+})