@outbox-event-bus/postgres-drizzle-outbox 1.0.0

package/src/postgres-drizzle-outbox.ts

@@ -0,0 +1,195 @@
+import { and, eq, inArray, lt, or, sql } from "drizzle-orm"
+import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"
+import {
+  type BusEvent,
+  type ErrorHandler,
+  EventStatus,
+  type FailedBusEvent,
+  formatErrorMessage,
+  type IOutbox,
+  type OutboxConfig,
+  PollingService,
+  reportEventError,
+} from "outbox-event-bus"
+import { outboxEvents, outboxEventsArchive } from "./schema"
+
+export interface PostgresDrizzleOutboxConfig extends OutboxConfig {
+  db: PostgresJsDatabase<Record<string, unknown>>
+  getTransaction?: (() => PostgresJsDatabase<Record<string, unknown>> | undefined) | undefined
+  tables?: {
+    outboxEvents: typeof outboxEvents
+    outboxEventsArchive: typeof outboxEventsArchive
+  }
+}
+
+export class PostgresDrizzleOutbox implements IOutbox<PostgresJsDatabase<Record<string, unknown>>> {
+  private readonly config: Required<PostgresDrizzleOutboxConfig>
+  private readonly poller: PollingService
+
+  constructor(config: PostgresDrizzleOutboxConfig) {
+    this.config = {
+      batchSize: config.batchSize ?? 50,
+      pollIntervalMs: config.pollIntervalMs ?? 1000,
+      maxRetries: config.maxRetries ?? 5,
+      baseBackoffMs: config.baseBackoffMs ?? 1000,
+      processingTimeoutMs: config.processingTimeoutMs ?? 30000,
+      maxErrorBackoffMs: config.maxErrorBackoffMs ?? 30000,
+      db: config.db,
+      getTransaction: config.getTransaction,
+      tables: config.tables ?? {
+        outboxEvents,
+        outboxEventsArchive,
+      },
+    }
+
+    this.poller = new PollingService({
+      pollIntervalMs: this.config.pollIntervalMs,
+      baseBackoffMs: this.config.baseBackoffMs,
+      maxErrorBackoffMs: this.config.maxErrorBackoffMs,
+      processBatch: (handler) => this.processBatch(handler),
+    })
+  }
+
+  async publish(
+    events: BusEvent[],
+    transaction?: PostgresJsDatabase<Record<string, unknown>>
+  ): Promise<void> {
+    const executor = transaction ?? this.config.getTransaction?.() ?? this.config.db
+
+    await executor.insert(this.config.tables.outboxEvents).values(
+      events.map((event) => ({
+        id: event.id,
+        type: event.type,
+        payload: event.payload,
+        occurredAt: event.occurredAt,
+        status: EventStatus.CREATED,
+      }))
+    )
+  }
+
+  async getFailedEvents(): Promise<FailedBusEvent[]> {
+    const events = await this.config.db
+      .select()
+      .from(this.config.tables.outboxEvents)
+      .where(eq(this.config.tables.outboxEvents.status, EventStatus.FAILED))
+      .orderBy(sql`${this.config.tables.outboxEvents.occurredAt} DESC`)
+      .limit(100)
+
+    return events.map((event) => {
+      const failedEvent: FailedBusEvent = {
+        id: event.id,
+        type: event.type,
+        payload: event.payload as any,
+        occurredAt: event.occurredAt,
+        retryCount: event.retryCount,
+      }
+      if (event.lastError) failedEvent.error = event.lastError
+      if (event.startedOn) failedEvent.lastAttemptAt = event.startedOn
+      return failedEvent
+    })
+  }
+
+  async retryEvents(eventIds: string[]): Promise<void> {
+    await this.config.db
+      .update(this.config.tables.outboxEvents)
+      .set({
+        status: EventStatus.CREATED,
+        retryCount: 0,
+        nextRetryAt: null,
+        lastError: null,
+      })
+      .where(inArray(this.config.tables.outboxEvents.id, eventIds))
+  }
+
+  start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void {
+    this.poller.start(handler, onError)
+  }
+
+  async stop(): Promise<void> {
+    await this.poller.stop()
+  }
+
+  private async processBatch(handler: (event: BusEvent) => Promise<void>) {
+    await this.config.db.transaction(async (transaction) => {
+      const now = new Date()
+
+      // Select events that are:
+      // 1. New (status = created)
+      // 2. Failed but can be retried (retry count < max AND retry time has passed)
+      // 3. Active but stuck/timed out (keepAlive is older than now - expireInSeconds)
+      const events = await transaction
+        .select()
+        .from(this.config.tables.outboxEvents)
+        .where(
+          or(
+            eq(this.config.tables.outboxEvents.status, EventStatus.CREATED),
+            and(
+              eq(this.config.tables.outboxEvents.status, EventStatus.FAILED),
+              lt(this.config.tables.outboxEvents.retryCount, this.config.maxRetries),
+              lt(this.config.tables.outboxEvents.nextRetryAt, now)
+            ),
+            and(
+              eq(this.config.tables.outboxEvents.status, EventStatus.ACTIVE),
+              // Check if event is stuck: keepAlive is older than (now - expireInSeconds)
+              // This uses PostgreSQL's make_interval to subtract expireInSeconds from current timestamp
+              lt(
+                this.config.tables.outboxEvents.keepAlive,
+                sql`${now.toISOString()}::timestamp - make_interval(secs => ${this.config.tables.outboxEvents.expireInSeconds})`
+              )
+            )
+          )
+        )
+        .limit(this.config.batchSize)
+        .for("update", { skipLocked: true })
+
+      if (events.length === 0) return
+
+      const eventIds = events.map((event) => event.id)
+
+      await transaction
+        .update(this.config.tables.outboxEvents)
+        .set({
+          status: EventStatus.ACTIVE,
+          startedOn: now,
+          keepAlive: now,
+        })
+        .where(inArray(this.config.tables.outboxEvents.id, eventIds))
+
+      for (const event of events) {
+        try {
+          await handler(event)
+          // Archive successful event immediately
+          await transaction.insert(this.config.tables.outboxEventsArchive).values({
+            id: event.id,
+            type: event.type,
+            payload: event.payload,
+            occurredAt: event.occurredAt,
+            status: EventStatus.COMPLETED,
+            retryCount: event.retryCount,
+            createdOn: event.createdOn,
+            startedOn: now,
+            completedOn: new Date(),
+          })
+          await transaction
+            .delete(this.config.tables.outboxEvents)
+            .where(eq(this.config.tables.outboxEvents.id, event.id))
+        } catch (error: unknown) {
+          const retryCount = event.retryCount + 1
+          reportEventError(this.poller.onError, error, event, retryCount, this.config.maxRetries)
+
+          // Mark this specific event as failed
+          const delay = this.poller.calculateBackoff(retryCount)
+          await transaction
+            .update(this.config.tables.outboxEvents)
+            .set({
+              status: EventStatus.FAILED,
+              retryCount,
+              lastError: formatErrorMessage(error),
+              nextRetryAt: new Date(Date.now() + delay),
+            })
+            .where(eq(this.config.tables.outboxEvents.id, event.id))
+        }
+      }
+    })
+  }
+}

package/src/schema.ts

@@ -0,0 +1,37 @@
+import { integer, jsonb, pgEnum, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"
+
+export const outboxStatusEnum = pgEnum("outbox_status", [
+  "created",
+  "active",
+  "completed",
+  "failed",
+])
+
+export const outboxEvents = pgTable("outbox_events", {
+  id: uuid("id").primaryKey(),
+  type: text("type").notNull(),
+  payload: jsonb("payload").notNull(),
+  occurredAt: timestamp("occurred_at").notNull(),
+  status: outboxStatusEnum("status").notNull().default("created"),
+  retryCount: integer("retry_count").notNull().default(0),
+  lastError: text("last_error"),
+  nextRetryAt: timestamp("next_retry_at"),
+  createdOn: timestamp("created_on").notNull().defaultNow(),
+  startedOn: timestamp("started_on"),
+  completedOn: timestamp("completed_on"),
+  keepAlive: timestamp("keep_alive"),
+  expireInSeconds: integer("expire_in_seconds").notNull().default(300),
+})
+
+export const outboxEventsArchive = pgTable("outbox_events_archive", {
+  id: uuid("id").primaryKey(),
+  type: text("type").notNull(),
+  payload: jsonb("payload").notNull(),
+  occurredAt: timestamp("occurred_at").notNull(),
+  status: outboxStatusEnum("status").notNull(),
+  retryCount: integer("retry_count").notNull(),
+  lastError: text("last_error"),
+  createdOn: timestamp("created_on").notNull(),
+  startedOn: timestamp("started_on"),
+  completedOn: timestamp("completed_on").notNull(),
+})

package/src/transaction-storage.ts

@@ -0,0 +1,21 @@
+import { AsyncLocalStorage } from "node:async_hooks"
+import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"
+
+export const drizzleTransactionStorage = new AsyncLocalStorage<
+  PostgresJsDatabase<Record<string, unknown>>
+>()
+
+export async function withDrizzleTransaction<T>(
+  db: PostgresJsDatabase<Record<string, unknown>>,
+  fn: (tx: PostgresJsDatabase<Record<string, unknown>>) => Promise<T>
+): Promise<T> {
+  return db.transaction(async (tx) => {
+    return drizzleTransactionStorage.run(tx, () => fn(tx))
+  })
+}
+
+export function getDrizzleTransaction(): () =>
+  | PostgresJsDatabase<Record<string, unknown>>
+  | undefined {
+  return () => drizzleTransactionStorage.getStore()
+}

package/src/transactions.e2e.ts

@@ -0,0 +1,142 @@
+import { AsyncLocalStorage } from "node:async_hooks"
+import { eq } from "drizzle-orm"
+import { pgTable, text, timestamp } from "drizzle-orm/pg-core"
+import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"
+import { drizzle } from "drizzle-orm/postgres-js"
+import { OutboxEventBus } from "outbox-event-bus"
+import postgres from "postgres"
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"
+import { PostgresDrizzleOutbox } from "./index"
+import { outboxEvents } from "./schema"
+
+// Define a dummy business table for the test
+const users = pgTable("users", {
+  id: text("id").primaryKey(),
+  name: text("name").notNull(),
+  createdAt: timestamp("created_at").defaultNow().notNull(),
+})
+
+type Db = PostgresJsDatabase<Record<string, never>>
+
+describe("PostgresDrizzle Outbox Transactions with AsyncLocalStorage", () => {
+  const sql = postgres("postgres://test_user:test_password@localhost:5433/outbox_test")
+  const db = drizzle(sql)
+
+  const als = new AsyncLocalStorage<Db>()
+
+  // A helper that acts as a sqlExecutor, grabbing the transaction from ALS if it exists
+  const _sqlExecutorProxy = new Proxy(db, {
+    get(target, prop, receiver) {
+      const transaction = als.getStore()
+      return Reflect.get(transaction ?? target, prop, receiver)
+    },
+  }) as unknown as Db
+
+  const outbox = new PostgresDrizzleOutbox({
+    db,
+    getTransaction: () => als.getStore(),
+    pollIntervalMs: 50,
+  })
+
+  // We use the proxy as the context for the event bus
+  const eventBus = new OutboxEventBus(
+    outbox,
+    (_bus, type, count) => console.warn(`Max listeners for ${type}: ${count}`),
+    (err) => console.error("Bus error:", err)
+  )
+
+  beforeAll(async () => {
+    // Schema setup (in a real app this would be migrations)
+    await sql`DROP TABLE IF EXISTS users`
+    await sql`DROP TABLE IF EXISTS outbox_events`
+    await sql`DROP TABLE IF EXISTS outbox_events_archive`
+    await sql`DROP TYPE IF EXISTS outbox_status`
+
+    await sql`CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, created_at TIMESTAMP DEFAULT NOW())`
+    await sql`
+      CREATE TYPE outbox_status AS ENUM ('created', 'active', 'completed', 'failed')
+    `
+    await sql`CREATE TABLE outbox_events (
+      id UUID PRIMARY KEY,
+      type TEXT NOT NULL,
+      payload JSONB NOT NULL,
+      occurred_at TIMESTAMP NOT NULL,
+      status outbox_status NOT NULL DEFAULT 'created',
+      retry_count INTEGER NOT NULL DEFAULT 0,
+      last_error TEXT,
+      next_retry_at TIMESTAMP,
+      created_on TIMESTAMP NOT NULL DEFAULT NOW(),
+      started_on TIMESTAMP,
+      completed_on TIMESTAMP,
+      keep_alive TIMESTAMP,
+      expire_in_seconds INTEGER NOT NULL DEFAULT 300
+    )`
+  })
+
+  beforeEach(async () => {
+    await sql`DELETE FROM users`
+    await sql`DELETE FROM outbox_events`
+  })
+
+  afterAll(async () => {
+    await sql.end()
+  })
+
+  it("should commit both business data and outbox event in a transaction", async () => {
+    const eventId = "3ed0f0a5-f4e1-4c7b-b5d1-1234567890ab"
+    const userId = "user_456"
+
+    await db.transaction(async (transaction) => {
+      await als.run(transaction, async () => {
+        // 1. Perform business operation
+        await transaction.insert(users).values({ id: userId, name: "Alice" })
+
+        // 2. Emit event using the transaction from ALS via our proxy
+        await eventBus.emit({
+          id: eventId,
+          type: "USER_CREATED",
+          payload: { userId, name: "Alice" },
+          occurredAt: new Date(),
+        })
+      })
+    })
+
+    // Verify both are persisted
+    const user = await db.select().from(users).where(eq(users.id, userId))
+    expect(user).toHaveLength(1)
+
+    const event = await db.select().from(outboxEvents).where(eq(outboxEvents.id, eventId))
+    expect(event).toHaveLength(1)
+  })
+
+  it("should rollback both business data and outbox event on failure", async () => {
+    const eventId = "4fd1f1b6-f5f2-4d8c-c6e2-2345678901bc"
+    const userId = "user_fail"
+
+    try {
+      await db.transaction(async (transaction) => {
+        await als.run(transaction, async () => {
+          await transaction.insert(users).values({ id: userId, name: "Bob" })
+
+          await eventBus.emit({
+            id: eventId,
+            type: "USER_CREATED",
+            payload: { userId, name: "Bob" },
+            occurredAt: new Date(),
+          })
+
+          throw new Error("Forced rollback")
+        })
+      })
+    } catch (_err) {
+      // Expected
+    }
+
+    // Verify neither are persisted
+    const user = await db.select().from(users).where(eq(users.id, userId))
+    expect(user).toHaveLength(0)
+
+    const event = await db.select().from(outboxEvents).where(eq(outboxEvents.id, eventId))
+    expect(event).toHaveLength(0)
+  })
+})