@outbox-event-bus/sqlite-better-sqlite3-outbox 1.0.0

package/src/integration.e2e.ts

@@ -0,0 +1,258 @@
+import { randomUUID } from "node:crypto"
+import { existsSync, unlinkSync } from "node:fs"
+import { join } from "node:path"
+import Database from "better-sqlite3"
+import { afterAll, beforeAll, describe, expect, it } from "vitest"
+import { SqliteBetterSqlite3Outbox } from "./sqlite-better-sqlite3-outbox"
+
+const DB_PATH = join(process.cwd(), `test-outbox-${randomUUID()}.db`)
+
+describe("SqliteBetterSqlite3Outbox E2E", () => {
+  let db: Database.Database
+
+  beforeAll(() => {
+    if (existsSync(DB_PATH)) {
+      unlinkSync(DB_PATH)
+    }
+    db = new Database(DB_PATH)
+  })
+
+  afterAll(() => {
+    if (db) {
+      db.close()
+    }
+    if (existsSync(DB_PATH)) {
+      unlinkSync(DB_PATH)
+    }
+  })
+
+  it("should process events end-to-end", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      pollIntervalMs: 100,
+    })
+
+    const eventId = "event-1"
+    const event = {
+      id: eventId,
+      type: "user.created",
+      payload: { userId: "123", email: "test@example.com" },
+      occurredAt: new Date(),
+    }
+
+    await outbox.publish([event])
+
+    const result = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId) as any
+    expect(result).toBeDefined()
+    expect(result.status).toBe("created")
+
+    const processedEvents: any[] = []
+    const handler = async (event: any) => {
+      processedEvents.push(event)
+    }
+
+    outbox.start(handler, (err) => console.error("Outbox Error:", err))
+
+    await new Promise((resolve) => setTimeout(resolve, 1000))
+
+    expect(processedEvents).toHaveLength(1)
+    expect(processedEvents[0].id).toBe(eventId)
+
+    const eventAfter = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId)
+    expect(eventAfter).toBeUndefined()
+
+    const archiveResult = db
+      .prepare("SELECT * FROM outbox_events_archive WHERE id = ?")
+      .get(eventId) as any
+    expect(archiveResult).toBeDefined()
+    expect(archiveResult.status).toBe("completed")
+
+    await outbox.stop()
+  })
+
+  it("should retry failed events", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      pollIntervalMs: 100,
+      baseBackoffMs: 100,
+    })
+
+    const eventId = "event-2"
+    const event = {
+      id: eventId,
+      type: "order.placed",
+      payload: { orderId: "abc" },
+      occurredAt: new Date(),
+    }
+
+    await outbox.publish([event])
+
+    let attempts = 0
+    const handler = async (_event: any) => {
+      attempts++
+      throw new Error("Processing failed")
+    }
+
+    outbox.start(handler, () => {})
+
+    await new Promise((resolve) => setTimeout(resolve, 1500))
+
+    await outbox.stop()
+
+    const result = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId) as any
+    expect(result).toBeDefined()
+    expect(result.status).toBe("failed")
+    expect(result.retry_count).toBeGreaterThan(1)
+    expect(attempts).toBeGreaterThan(1)
+  })
+
+  it("should support manual management of failed events", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      pollIntervalMs: 100,
+    })
+
+    const eventId = "event-manual"
+    const event = {
+      id: eventId,
+      type: "manual.retry",
+      payload: {},
+      occurredAt: new Date(),
+    }
+
+    // 1. Insert directly as failed
+    db.prepare(`
+        INSERT INTO outbox_events (id, type, payload, occurred_at, status, retry_count, last_error)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      eventId,
+      event.type,
+      JSON.stringify(event.payload),
+      event.occurredAt.toISOString(),
+      "failed",
+      5,
+      "Manual failure"
+    )
+
+    const inserted = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId)
+
+    const failed = await outbox.getFailedEvents()
+    const targetEvent = failed.find((e) => e.id === eventId)
+
+    expect(targetEvent).toBeDefined()
+    expect(targetEvent!.id).toBe(eventId)
+    expect(targetEvent!.error).toBe("Manual failure")
+    expect(targetEvent!.retryCount).toBe(5)
+
+    await outbox.retryEvents([eventId])
+
+    const retried = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId) as any
+    expect(retried).toBeDefined()
+    expect(retried.status).toBe("created")
+    expect(retried.retry_count).toBe(0)
+    expect(retried.last_error).toBeNull()
+
+    const processed: any[] = []
+    outbox.start(
+      async (e) => {
+        processed.push(e)
+      },
+      (err) => console.error(err)
+    )
+
+    await new Promise((r) => setTimeout(r, 1000))
+    await outbox.stop()
+
+    const processedEvent = processed.find((e) => e.id === eventId)
+    expect(processedEvent).toBeDefined()
+    expect(processedEvent!.id).toBe(eventId)
+  })
+
+  it("should recover from stuck events", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      pollIntervalMs: 100,
+    })
+
+    const eventId = "event-stuck"
+    const now = new Date()
+
+    db.prepare(`
+      INSERT INTO outbox_events (id, type, payload, occurred_at, status, retry_count, keep_alive, expire_in_seconds)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      eventId,
+      "stuck.event",
+      JSON.stringify({ stuck: true }),
+      new Date(now.getTime() - 400000).toISOString(),
+      "active",
+      0,
+      new Date(now.getTime() - 350000).toISOString(),
+      300
+    )
+
+    const processedEvents: any[] = []
+    const handler = async (event: any) => {
+      processedEvents.push(event)
+    }
+
+    outbox.start(handler, (err) => console.error("Outbox Error:", err))
+
+    await new Promise((resolve) => setTimeout(resolve, 1500))
+
+    expect(processedEvents.some((e) => e.id === eventId)).toBe(true)
+
+    await outbox.stop()
+  })
+
+  it("should handle concurrent processing safely", async () => {
+    const eventCount = 50
+    const events = Array.from({ length: eventCount }).map((_, i) => ({
+      id: `concurrent-${i}`,
+      type: "concurrent.test",
+      payload: { index: i },
+      occurredAt: new Date(),
+    }))
+
+    const outboxPublisher = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      pollIntervalMs: 100,
+    })
+    await outboxPublisher.publish(events)
+    await outboxPublisher.stop()
+
+    const workerCount = 5
+    const processedEvents: any[] = []
+    const workers: SqliteBetterSqlite3Outbox[] = []
+
+    // Shared handler that pushes to processedEvents
+    const handler = async (event: any) => {
+      await new Promise((resolve) => setTimeout(resolve, Math.random() * 50))
+      processedEvents.push(event)
+    }
+
+    for (let i = 0; i < workerCount; i++) {
+      const worker = new SqliteBetterSqlite3Outbox({
+        dbPath: DB_PATH,
+        pollIntervalMs: 100 + Math.random() * 50,
+        batchSize: 5,
+      })
+      workers.push(worker)
+      worker.start(handler, (err) => console.error(`Worker ${i} Error:`, err))
+    }
+
+    const maxWaitTime = 10000
+    const startTime = Date.now()
+
+    while (processedEvents.length < eventCount && Date.now() - startTime < maxWaitTime) {
+      await new Promise((resolve) => setTimeout(resolve, 200))
+    }
+
+    await Promise.all(workers.map((w) => w.stop()))
+
+    expect(processedEvents).toHaveLength(eventCount)
+    const ids = processedEvents.map((event) => event.id)
+    const uniqueIds = new Set(ids)
+    expect(uniqueIds.size).toBe(eventCount)
+  })
+})

package/src/schema.sql

@@ -0,0 +1,30 @@
+CREATE TABLE IF NOT EXISTS outbox_events (
+  id TEXT PRIMARY KEY,
+  type TEXT NOT NULL,
+  payload TEXT NOT NULL,
+  occurred_at TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'created',
+  retry_count INTEGER NOT NULL DEFAULT 0,
+  last_error TEXT,
+  next_retry_at TEXT,
+  created_on TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  started_on TEXT,
+  completed_on TEXT,
+  keep_alive TEXT,
+  expire_in_seconds INTEGER NOT NULL DEFAULT 300
+);
+
+CREATE TABLE IF NOT EXISTS outbox_events_archive (
+  id TEXT PRIMARY KEY,
+  type TEXT NOT NULL,
+  payload TEXT NOT NULL,
+  occurred_at TEXT NOT NULL,
+  status TEXT NOT NULL,
+  retry_count INTEGER NOT NULL,
+  last_error TEXT,
+  created_on TEXT NOT NULL,
+  started_on TEXT,
+  completed_on TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_outbox_events_status_retry ON outbox_events (status, next_retry_at);

package/src/sqlite-better-sqlite3-outbox.ts

@@ -0,0 +1,291 @@
+import Database from "better-sqlite3"
+import {
+  type BusEvent,
+  type ErrorHandler,
+  EventStatus,
+  type FailedBusEvent,
+  formatErrorMessage,
+  type IOutbox,
+  type OutboxConfig,
+  PollingService,
+  reportEventError,
+} from "outbox-event-bus"
+
+const DEFAULT_EXPIRE_IN_SECONDS = 300
+
+const getOutboxSchema = (tableName: string, archiveTableName: string) => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    id TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    occurred_at TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT '${EventStatus.CREATED}',
+    retry_count INTEGER NOT NULL DEFAULT 0,
+    last_error TEXT,
+    next_retry_at TEXT,
+    created_on TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+    started_on TEXT,
+    completed_on TEXT,
+    keep_alive TEXT,
+    expire_in_seconds INTEGER NOT NULL DEFAULT ${DEFAULT_EXPIRE_IN_SECONDS}
+  );
+
+  CREATE TABLE IF NOT EXISTS ${archiveTableName} (
+    id TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    occurred_at TEXT NOT NULL,
+    status TEXT NOT NULL,
+    retry_count INTEGER NOT NULL,
+    last_error TEXT,
+    created_on TEXT NOT NULL,
+    started_on TEXT,
+    completed_on TEXT NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_${tableName}_status_retry ON ${tableName} (status, next_retry_at);
+`
+
+export interface SqliteBetterSqlite3OutboxConfig extends OutboxConfig {
+  dbPath?: string
+  db?: Database.Database
+  getTransaction?: (() => Database.Database | undefined) | undefined
+  tableName?: string
+  archiveTableName?: string
+}
+
+interface OutboxRow {
+  id: string
+  type: string
+  payload: string
+  occurred_at: string
+  status: EventStatus
+  retry_count: number
+  next_retry_at: string | null
+  last_error: string | null
+  created_on: string
+  started_on: string | null
+  completed_on: string | null
+  keep_alive: string | null
+  expire_in_seconds: number
+}
+
+export class SqliteBetterSqlite3Outbox implements IOutbox<Database.Database> {
+  private readonly config: Required<SqliteBetterSqlite3OutboxConfig>
+  private readonly db: Database.Database
+  private readonly poller: PollingService
+
+  constructor(config: SqliteBetterSqlite3OutboxConfig) {
+    if (config.db) {
+      this.db = config.db
+    } else {
+      if (!config.dbPath) throw new Error("dbPath is required if db is not provided")
+      this.db = new Database(config.dbPath)
+      this.db.pragma("journal_mode = WAL")
+    }
+
+    this.config = {
+      batchSize: config.batchSize ?? 50,
+      pollIntervalMs: config.pollIntervalMs ?? 1000,
+      maxRetries: config.maxRetries ?? 5,
+      baseBackoffMs: config.baseBackoffMs ?? 1000,
+      processingTimeoutMs: config.processingTimeoutMs ?? 30000,
+      maxErrorBackoffMs: config.maxErrorBackoffMs ?? 30000,
+      dbPath: config.dbPath ?? "",
+      db: this.db,
+      getTransaction: config.getTransaction,
+      tableName: config.tableName ?? "outbox_events",
+      archiveTableName: config.archiveTableName ?? "outbox_events_archive",
+    } as Required<SqliteBetterSqlite3OutboxConfig>
+
+    this.init()
+
+    this.poller = new PollingService({
+      pollIntervalMs: this.config.pollIntervalMs,
+      baseBackoffMs: this.config.baseBackoffMs,
+      maxErrorBackoffMs: this.config.maxErrorBackoffMs,
+      processBatch: (handler) => this.processBatch(handler),
+    })
+  }
+
+  private init() {
+    this.db.exec(getOutboxSchema(this.config.tableName, this.config.archiveTableName))
+  }
+
+  async publish(events: BusEvent[], transaction?: Database.Database): Promise<void> {
+    if (events.length === 0) return
+
+    const executor = transaction ?? this.config.getTransaction?.() ?? this.db
+
+    const insert = executor.prepare(`
+      INSERT INTO ${this.config.tableName} (id, type, payload, occurred_at, status)
+      VALUES (?, ?, ?, ?, '${EventStatus.CREATED}')
+    `)
+
+    executor.transaction(() => {
+      for (const event of events) {
+        insert.run(
+          event.id,
+          event.type,
+          JSON.stringify(event.payload),
+          event.occurredAt.toISOString()
+        )
+      }
+    })()
+  }
+
+  async getFailedEvents(): Promise<FailedBusEvent[]> {
+    const rows = this.db
+      .prepare(`
+      SELECT * FROM ${this.config.tableName}
+      WHERE status = '${EventStatus.FAILED}'
+      ORDER BY occurred_at DESC
+      LIMIT 100
+    `)
+      .all() as OutboxRow[]
+
+    return rows.map((row) => {
+      const event: FailedBusEvent = {
+        id: row.id,
+        type: row.type,
+        payload: JSON.parse(row.payload),
+        occurredAt: new Date(row.occurred_at),
+        retryCount: row.retry_count,
+      }
+      if (row.last_error) event.error = row.last_error
+      if (row.started_on) event.lastAttemptAt = new Date(row.started_on)
+      return event
+    })
+  }
+
+  async retryEvents(eventIds: string[]): Promise<void> {
+    if (eventIds.length === 0) return
+
+    const placeholders = eventIds.map(() => "?").join(",")
+    this.db
+      .prepare(`
+      UPDATE ${this.config.tableName}
+      SET status = '${EventStatus.CREATED}',
+          retry_count = 0,
+          next_retry_at = NULL,
+          last_error = NULL
+      WHERE id IN (${placeholders})
+    `)
+      .run(...eventIds)
+  }
+
+  start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void {
+    this.poller.start(handler, onError)
+  }
+
+  async stop(): Promise<void> {
+    await this.poller.stop()
+  }
+
+  private async processBatch(handler: (event: BusEvent) => Promise<void>) {
+    const now = new Date().toISOString()
+    const msNow = Date.now()
+
+    const lockedEvents = this.db.transaction(() => {
+      // Select events that are ready to process:
+      // 1. New events (status = 'created')
+      // 2. Failed events that can be retried (retry_count < max AND next_retry_at has passed)
+      // 3. Stuck events (status = 'active' but keepAlive + expire_in_seconds < now)
+      const rows = this.db
+        .prepare(`
+        SELECT * FROM ${this.config.tableName}
+        WHERE status = '${EventStatus.CREATED}'
+        OR (status = '${EventStatus.FAILED}' AND retry_count < ? AND next_retry_at <= ?)
+        OR (status = '${EventStatus.ACTIVE}' AND datetime(keep_alive, '+' || expire_in_seconds || ' seconds') < datetime(?))
+        LIMIT ?
+      `)
+        .all(this.config.maxRetries, now, now, this.config.batchSize) as OutboxRow[]
+
+      if (rows.length === 0) return []
+
+      const ids = rows.map((r) => r.id)
+      const placeholders = ids.map(() => "?").join(",")
+
+      this.db
+        .prepare(`
+        UPDATE ${this.config.tableName}
+        SET status = '${EventStatus.ACTIVE}',
+            started_on = ?,
+            keep_alive = ?
+        WHERE id IN (${placeholders})
+      `)
+        .run(now, now, ...ids)
+
+      return rows
+    })()
+
+    if (lockedEvents.length === 0) return
+
+    const busEvents: BusEvent[] = lockedEvents.map((row) => ({
+      id: row.id,
+      type: row.type,
+      payload: JSON.parse(row.payload),
+      occurredAt: new Date(row.occurred_at),
+    }))
+
+    const completedEvents: { event: BusEvent; lockedEvent: OutboxRow }[] = []
+
+    for (let index = 0; index < busEvents.length; index++) {
+      const event = busEvents[index]!
+      const lockedEvent = lockedEvents[index]!
+
+      try {
+        await handler(event)
+        completedEvents.push({ event, lockedEvent })
+      } catch (error) {
+        const retryCount = lockedEvent.retry_count + 1
+        reportEventError(this.poller.onError, error, event, retryCount, this.config.maxRetries)
+
+        const delay = this.poller.calculateBackoff(retryCount)
+
+        this.db
+          .prepare(`
+          UPDATE ${this.config.tableName}
+          SET status = '${EventStatus.FAILED}',
+              retry_count = ?,
+              last_error = ?,
+              next_retry_at = ?
+          WHERE id = ?
+        `)
+          .run(
+            retryCount,
+            formatErrorMessage(error),
+            new Date(msNow + delay).toISOString(),
+            lockedEvent.id
+          )
+      }
+    }
+
+    if (completedEvents.length > 0) {
+      this.db.transaction(() => {
+        const insertArchive = this.db.prepare(`
+          INSERT INTO ${this.config.archiveTableName} (
+            id, type, payload, occurred_at, status, retry_count, last_error, created_on, started_on, completed_on
+          ) VALUES (?, ?, ?, ?, '${EventStatus.COMPLETED}', ?, ?, ?, ?, ?)
+        `)
+        const deleteEvent = this.db.prepare(`DELETE FROM ${this.config.tableName} WHERE id = ?`)
+
+        const completionTime = new Date().toISOString()
+        for (const { lockedEvent } of completedEvents) {
+          insertArchive.run(
+            lockedEvent.id,
+            lockedEvent.type,
+            lockedEvent.payload,
+            lockedEvent.occurred_at,
+            lockedEvent.retry_count,
+            lockedEvent.last_error,
+            lockedEvent.created_on,
+            now,
+            completionTime
+          )
+          deleteEvent.run(lockedEvent.id)
+        }
+      })()
+    }
+  }
+}

package/src/sync-transaction.test.ts

@@ -0,0 +1,117 @@
+import { existsSync, unlinkSync } from "node:fs"
+import Database from "better-sqlite3"
+import { afterAll, beforeEach, describe, expect, it } from "vitest"
+import { OutboxEventBus } from "../../../core/src/bus/outbox-event-bus"
+import { SqliteBetterSqlite3Outbox } from "./index"
+
+const DB_PATH = "./test-sync-transactions.db"
+
+describe("SqliteBetterSqlite3Outbox Synchronous Transactions", () => {
+  let db: Database.Database
+
+  beforeEach(() => {
+    db = new Database(":memory:")
+    // Create business table
+    db.exec("CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, name TEXT)")
+  })
+
+  afterAll(() => {
+    if (db) {
+      db.close()
+    }
+    if (existsSync(DB_PATH)) {
+      unlinkSync(DB_PATH)
+    }
+  })
+
+  it("should atomically commit event and data when using shared connection", async () => {
+    // 1. Setup Outbox with getTransaction returning the shared db
+    const outbox = new SqliteBetterSqlite3Outbox({
+      db,
+      getTransaction: () => db, // <--- CRITICAL: Use the same DB instance
+    })
+    const bus = new OutboxEventBus(outbox, (err) => console.error(err))
+
+    const userId = "user-1"
+    const eventId = "event-1"
+
+    // 2. Run synchronous transaction
+    const createUser = db.transaction(() => {
+      db.prepare("INSERT INTO users (id, name) VALUES (?, ?)").run(userId, "Alice")
+
+      // Emit event synchronously (using void to ignore Promise)
+      // Since we provided getTransaction: () => db, this uses the SAME connection
+      // and thus participates in the transaction via SAVEPOINT
+      void bus.emit({
+        id: eventId,
+        type: "user.created",
+        payload: { id: userId, name: "Alice" },
+      })
+    })
+
+    createUser()
+
+    // 3. Verify both exist
+    const user = db.prepare("SELECT * FROM users WHERE id = ?").get(userId)
+    expect(user).toBeDefined()
+
+    const event = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId)
+    expect(event).toBeDefined()
+  })
+
+  it("should atomically rollback event and data on error", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      db,
+      getTransaction: () => db,
+    })
+    const bus = new OutboxEventBus(outbox, (err) => console.error(err))
+
+    const userId = "user-2"
+    const eventId = "event-2"
+
+    // 2. Run synchronous transaction that fails
+    const createUser = db.transaction(() => {
+      db.prepare("INSERT INTO users (id, name) VALUES (?, ?)").run(userId, "Bob")
+
+      void bus.emit({
+        id: eventId,
+        type: "user.created",
+        payload: { id: userId, name: "Bob" },
+      })
+
+      throw new Error("Rollback!")
+    })
+
+    expect(() => createUser()).toThrow("Rollback!")
+
+    // 3. Verify NEITHER exist
+    const user = db.prepare("SELECT * FROM users WHERE id = ?").get(userId)
+    expect(user).toBeUndefined()
+
+    const event = db.prepare("SELECT * FROM outbox_events WHERE id = ?").get(eventId)
+    expect(event).toBeUndefined()
+  })
+
+  it("should throw TypeError if async function is used (preventing await usage)", async () => {
+    const outbox = new SqliteBetterSqlite3Outbox({
+      dbPath: DB_PATH,
+      getTransaction: () => db,
+    })
+    const _bus = new OutboxEventBus(outbox, (err) => console.error(err))
+
+    const userId = "user-async"
+
+    // @ts-expect-error - Testing runtime behavior
+    const tx = db.transaction(async () => {
+      db.prepare("INSERT INTO users (id, name) VALUES (?, ?)").run(userId, "Async")
+      await new Promise((resolve) => setTimeout(resolve, 10))
+    })
+
+    // It throws TypeError: Transaction function cannot return a promise
+    expect(() => tx()).toThrow("Transaction function cannot return a promise")
+
+    // Verify rollback happened
+    const user = db.prepare("SELECT * FROM users WHERE id = ?").get(userId)
+    expect(user).toBeUndefined()
+  })
+})