@dotdo/postgres 0.1.0

package/src/streaming/live-cdc-stream.ts

@@ -0,0 +1,985 @@
+/**
+ * Live CDC Stream - Real CDC streaming using PGLite's live.changes()
+ *
+ * This module provides true CDC (Change Data Capture) streaming by wrapping
+ * PGLite's built-in live.changes() API. Unlike QueryStreamer which simulates
+ * cursors by buffering results, this provides real streaming of changes as
+ * they happen.
+ *
+ * ## Architecture
+ *
+ * PGLite's live.changes() works by:
+ * 1. Creating a temporary view of the query
+ * 2. Setting up triggers on underlying tables
+ * 3. Using PostgreSQL's LISTEN/NOTIFY for change detection
+ * 4. Computing diffs between state snapshots
+ *
+ * This module wraps that functionality and provides:
+ * - Async iterator interface for consuming changes
+ * - HTTP/SSE streaming integration
+ * - Integration with pglake's WAL-to-Iceberg transformer
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * // Consume changes as async iterator
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ *
+ * // Or get an SSE stream for HTTP response
+ * const response = new Response(cdc.toSSEStream(), {
+ *   headers: { 'Content-Type': 'text/event-stream' }
+ * })
+ * ```
+ *
+ * @module streaming/live-cdc-stream
+ */
+
+// ============================================================================
+// Type Definitions
+// ============================================================================
+
+/**
+ * CDC operation type matching PGLite's live.changes() operations
+ */
+export type CDCOperationType = 'INSERT' | 'UPDATE' | 'DELETE' | 'RESET'
+
+/**
+ * Base change event interface
+ */
+export interface CDCChangeBase {
+  /** Operation type */
+  __op__: CDCOperationType
+  /** Columns that changed (for UPDATE) */
+  __changed_columns__: string[]
+  /** Position hint for ordering */
+  __after__?: number
+}
+
+/**
+ * INSERT change event
+ */
+export interface CDCInsertChange<_T = Record<string, unknown>> extends CDCChangeBase {
+  __op__: 'INSERT'
+  __after__: number
+}
+
+/**
+ * UPDATE change event
+ */
+export interface CDCUpdateChange<_T = Record<string, unknown>> extends CDCChangeBase {
+  __op__: 'UPDATE'
+  __after__: number
+}
+
+/**
+ * DELETE change event
+ */
+export interface CDCDeleteChange<_T = Record<string, unknown>> extends Omit<CDCChangeBase, '__after__'> {
+  __op__: 'DELETE'
+}
+
+/**
+ * RESET change event (full table refresh needed)
+ */
+export interface CDCResetChange extends CDCChangeBase {
+  __op__: 'RESET'
+  __changed_columns__: []
+}
+
+/**
+ * Union type for all change events
+ */
+export type CDCChange<T = Record<string, unknown>> =
+  | (CDCInsertChange<T> & T)
+  | (CDCUpdateChange<T> & T)
+  | (CDCDeleteChange<T> & T)
+  | CDCResetChange
+
+/**
+ * PGLite interface with live extension
+ */
+export interface PGLiteWithLive {
+  live: LiveNamespace
+}
+
+/**
+ * LiveNamespace from PGLite
+ */
+export interface LiveNamespace {
+  changes<T = Record<string, unknown>>(
+    query: string | LiveChangesOptions<T>,
+    params?: unknown[] | null,
+    key?: string,
+    callback?: (changes: CDCChange<T>[]) => void
+  ): Promise<LiveChangesHandle<T>>
+}
+
+/**
+ * Options for live.changes()
+ */
+export interface LiveChangesOptions<T = Record<string, unknown>> {
+  query: string
+  params?: unknown[] | null
+  key: string
+  callback?: (changes: CDCChange<T>[]) => void
+  signal?: AbortSignal
+}
+
+/**
+ * Handle returned by live.changes()
+ */
+export interface LiveChangesHandle<T = Record<string, unknown>> {
+  fields: Array<{ name: string; dataTypeID: number }>
+  initialChanges: CDCChange<T>[]
+  subscribe: (callback: (changes: CDCChange<T>[]) => void) => void
+  unsubscribe: (callback?: (changes: CDCChange<T>[]) => void) => Promise<void>
+  refresh: () => Promise<void>
+}
+
+/**
+ * Configuration for LiveCDCStream
+ */
+export interface LiveCDCStreamConfig<T = Record<string, unknown>> {
+  /** SQL query to watch */
+  query: string
+  /** Query parameters */
+  params?: unknown[] | null
+  /** Primary key column for change tracking (required) */
+  key: string
+  /** AbortSignal for cancellation */
+  signal?: AbortSignal
+  /** Batch size for buffering changes (default: 1) */
+  batchSize?: number
+  /** Flush interval for batched changes in ms (default: 0 = immediate) */
+  flushIntervalMs?: number
+  /** Transform function applied to each change */
+  transform?: (change: CDCChange<T>) => CDCChange<T> | null
+  /** Filter function to selectively process changes */
+  filter?: (change: CDCChange<T>) => boolean
+  /** Whether to include initial data on startup (default: true) */
+  includeInitial?: boolean
+  /** High water mark for buffering (default: 1000 changes) */
+  highWaterMark?: number
+}
+
+/**
+ * CDC stream statistics
+ */
+export interface CDCStreamStats {
+  /** Total changes processed */
+  totalChanges: number
+  /** Inserts processed */
+  insertCount: number
+  /** Updates processed */
+  updateCount: number
+  /** Deletes processed */
+  deleteCount: number
+  /** Resets processed */
+  resetCount: number
+  /** Changes filtered out */
+  filteredCount: number
+  /** Current buffer size */
+  bufferSize: number
+  /** Stream start time */
+  startedAt: number
+  /** Last change timestamp */
+  lastChangeAt: number | null
+  /** Changes per second (rolling average) */
+  changesPerSecond: number
+  /** Whether stream is active */
+  isActive: boolean
+}
+
+/**
+ * CDC event for Iceberg integration
+ */
+export interface IcebergCDCEvent<T = Record<string, unknown>> {
+  /** Log sequence number (monotonic) */
+  lsn: string
+  /** Operation type */
+  operation: 'INSERT' | 'UPDATE' | 'DELETE'
+  /** Schema name */
+  schema: string
+  /** Table name */
+  table: string
+  /** Primary key values */
+  primaryKey?: Record<string, unknown>
+  /** New row data (for INSERT/UPDATE) */
+  newRow?: T
+  /** Old row data (for UPDATE/DELETE) */
+  oldRow?: T
+  /** Changed columns (for UPDATE) */
+  changedColumns: string[]
+  /** Timestamp */
+  timestamp: number
+  /** Source shard ID */
+  shardId: string
+}
+
+// ============================================================================
+// Default Configuration
+// ============================================================================
+
+const DEFAULT_BATCH_SIZE = 1
+const DEFAULT_FLUSH_INTERVAL_MS = 0
+const DEFAULT_HIGH_WATER_MARK = 1000
+
+// ============================================================================
+// LiveCDCStream Class
+// ============================================================================
+
+/**
+ * LiveCDCStream provides real CDC streaming using PGLite's live.changes() API.
+ *
+ * This is the preferred way to stream changes from PostgreSQL in Durable Objects.
+ * It uses PGLite's built-in change detection mechanism which is more efficient
+ * than polling or simulated cursors.
+ *
+ * @example Basic usage
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * await stream.start()
+ *
+ * for await (const change of stream) {
+ *   console.log(`${change.__op__}: ${JSON.stringify(change)}`)
+ * }
+ * ```
+ *
+ * @example With filtering and transformation
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM orders WHERE status = $1',
+ *   params: ['pending'],
+ *   key: 'id',
+ *   filter: (change) => change.__op__ !== 'DELETE',
+ *   transform: (change) => ({
+ *     ...change,
+ *     processedAt: Date.now(),
+ *   }),
+ * })
+ * ```
+ */
+export class LiveCDCStream<T = Record<string, unknown>> implements AsyncIterable<CDCChange<T>> {
+  private readonly pg: PGLiteWithLive
+  private readonly config: Required<Omit<LiveCDCStreamConfig<T>, 'signal' | 'transform' | 'filter'>> &
+    Pick<LiveCDCStreamConfig<T>, 'signal' | 'transform' | 'filter'>
+  private handle: LiveChangesHandle<T> | null = null
+  private buffer: CDCChange<T>[] = []
+  private resolveWaiter: ((value: CDCChange<T> | null) => void) | null = null
+  private aborted = false
+  private started = false
+  private stats: CDCStreamStats
+  private lsnCounter = 0
+
+  constructor(pg: PGLiteWithLive, config: LiveCDCStreamConfig<T>) {
+    this.pg = pg
+    this.config = {
+      query: config.query,
+      params: config.params ?? null,
+      key: config.key,
+      batchSize: config.batchSize ?? DEFAULT_BATCH_SIZE,
+      flushIntervalMs: config.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS,
+      includeInitial: config.includeInitial ?? true,
+      highWaterMark: config.highWaterMark ?? DEFAULT_HIGH_WATER_MARK,
+      ...(config.signal !== undefined && { signal: config.signal }),
+      ...(config.transform !== undefined && { transform: config.transform }),
+      ...(config.filter !== undefined && { filter: config.filter }),
+    }
+
+    this.stats = {
+      totalChanges: 0,
+      insertCount: 0,
+      updateCount: 0,
+      deleteCount: 0,
+      resetCount: 0,
+      filteredCount: 0,
+      bufferSize: 0,
+      startedAt: 0,
+      lastChangeAt: null,
+      changesPerSecond: 0,
+      isActive: false,
+    }
+
+    // Handle abort signal
+    if (this.config.signal) {
+      this.config.signal.addEventListener('abort', () => {
+        this.abort()
+      }, { once: true })
+    }
+  }
+
+  /**
+   * Start the CDC stream.
+   *
+   * Must be called before iterating. Returns the initial changes if
+   * includeInitial is true (default).
+   */
+  async start(): Promise<CDCChange<T>[]> {
+    if (this.started) {
+      throw new Error('LiveCDCStream already started')
+    }
+
+    this.started = true
+    this.stats.startedAt = Date.now()
+    this.stats.isActive = true
+
+    // Handle pre-aborted signal
+    if (this.config.signal?.aborted) {
+      this.aborted = true
+      this.stats.isActive = false
+      return []
+    }
+
+    const liveOptions: LiveChangesOptions<T> = {
+      query: this.config.query,
+      params: this.config.params,
+      key: this.config.key,
+      callback: (changes) => this.handleChanges(changes),
+    }
+    if (this.config.signal !== undefined) {
+      liveOptions.signal = this.config.signal
+    }
+    this.handle = await this.pg.live.changes<T>(liveOptions)
+
+    // Process initial changes and add to buffer for stream consumption
+    const initialChanges = this.config.includeInitial
+      ? this.processChanges(this.handle.initialChanges)
+      : []
+
+    // Add initial changes to buffer so streams can consume them
+    for (const change of initialChanges) {
+      this.buffer.push(change)
+    }
+
+    return initialChanges
+  }
+
+  /**
+   * Stop the CDC stream and clean up resources.
+   */
+  async stop(): Promise<void> {
+    if (!this.handle) return
+
+    this.stats.isActive = false
+    this.aborted = true
+
+    // Resolve any pending waiter with null to signal end
+    if (this.resolveWaiter) {
+      this.resolveWaiter(null)
+      this.resolveWaiter = null
+          }
+
+    await this.handle.unsubscribe()
+    this.handle = null
+  }
+
+  /**
+   * Manually trigger a refresh of the underlying query.
+   */
+  async refresh(): Promise<void> {
+    if (!this.handle) {
+      throw new Error('LiveCDCStream not started')
+    }
+    await this.handle.refresh()
+  }
+
+  /**
+   * Get current stream statistics.
+   */
+  getStats(): CDCStreamStats {
+    // Calculate changes per second
+    if (this.stats.startedAt > 0) {
+      const elapsedSeconds = (Date.now() - this.stats.startedAt) / 1000
+      if (elapsedSeconds > 0) {
+        this.stats.changesPerSecond = this.stats.totalChanges / elapsedSeconds
+      }
+    }
+    this.stats.bufferSize = this.buffer.length
+
+    return { ...this.stats }
+  }
+
+  /**
+   * Get the field metadata from the query.
+   */
+  getFields(): Array<{ name: string; dataTypeID: number }> {
+    if (!this.handle) {
+      throw new Error('LiveCDCStream not started')
+    }
+    return this.handle.fields
+  }
+
+  /**
+   * Check if stream is currently active.
+   */
+  get isActive(): boolean {
+    return this.stats.isActive && !this.aborted
+  }
+
+  /**
+   * Async iterator implementation for consuming changes.
+   */
+  async *[Symbol.asyncIterator](): AsyncGenerator<CDCChange<T>, void, undefined> {
+    if (!this.started) {
+      await this.start()
+    }
+
+    while (!this.aborted) {
+      // First drain the buffer
+      while (this.buffer.length > 0) {
+        const change = this.buffer.shift()
+        if (change) {
+          yield change
+        }
+      }
+
+      // Wait for next change or abort
+      const change = await this.waitForChange()
+      if (change === null) {
+        break
+      }
+      yield change
+    }
+  }
+
+  /**
+   * Create a ReadableStream of CDC changes for HTTP streaming.
+   */
+  toReadableStream(): ReadableStream<CDCChange<T>> {
+    const self = this
+    let started = false
+
+    return new ReadableStream<CDCChange<T>>({
+      async start() {
+        if (!started && !self.started) {
+          await self.start()
+          started = true
+        }
+      },
+
+      async pull(controller) {
+        if (self.aborted) {
+          controller.close()
+          return
+        }
+
+        // Drain buffer first
+        while (self.buffer.length > 0) {
+          const change = self.buffer.shift()
+          if (change) {
+            controller.enqueue(change)
+            return
+          }
+        }
+
+        // Wait for next change
+        const change = await self.waitForChange()
+        if (change === null) {
+          controller.close()
+        } else {
+          controller.enqueue(change)
+        }
+      },
+
+      cancel() {
+        self.abort()
+      },
+    })
+  }
+
+  /**
+   * Create an SSE (Server-Sent Events) stream for HTTP streaming.
+   */
+  toSSEStream(): ReadableStream<Uint8Array> {
+    const self = this
+    const encoder = new TextEncoder()
+    let eventId = 0
+    let started = false
+
+    return new ReadableStream<Uint8Array>({
+      async start() {
+        if (!started && !self.started) {
+          await self.start()
+          started = true
+        }
+      },
+
+      async pull(controller) {
+        if (self.aborted) {
+          controller.enqueue(encoder.encode(`event: done\ndata: {}\n\n`))
+          controller.close()
+          return
+        }
+
+        // Drain buffer first
+        while (self.buffer.length > 0) {
+          const change = self.buffer.shift()
+          if (change) {
+            eventId++
+            const sseMessage = formatSSEMessage(change, eventId)
+            controller.enqueue(encoder.encode(sseMessage))
+            return
+          }
+        }
+
+        // Wait for next change
+        const change = await self.waitForChange()
+        if (change === null) {
+          controller.enqueue(encoder.encode(`event: done\ndata: {}\n\n`))
+          controller.close()
+        } else {
+          eventId++
+          const sseMessage = formatSSEMessage(change, eventId)
+          controller.enqueue(encoder.encode(sseMessage))
+        }
+      },
+
+      cancel() {
+        self.abort()
+      },
+    })
+  }
+
+  /**
+   * Create an NDJSON stream for HTTP streaming.
+   */
+  toNDJSONStream(): ReadableStream<Uint8Array> {
+    const self = this
+    const encoder = new TextEncoder()
+    let started = false
+
+    return new ReadableStream<Uint8Array>({
+      async start() {
+        if (!started && !self.started) {
+          await self.start()
+          started = true
+        }
+      },
+
+      async pull(controller) {
+        if (self.aborted) {
+          controller.close()
+          return
+        }
+
+        // Drain buffer first
+        while (self.buffer.length > 0) {
+          const change = self.buffer.shift()
+          if (change) {
+            const line = JSON.stringify(change) + '\n'
+            controller.enqueue(encoder.encode(line))
+            return
+          }
+        }
+
+        // Wait for next change
+        const change = await self.waitForChange()
+        if (change === null) {
+          controller.close()
+        } else {
+          const line = JSON.stringify(change) + '\n'
+          controller.enqueue(encoder.encode(line))
+        }
+      },
+
+      cancel() {
+        self.abort()
+      },
+    })
+  }
+
+  /**
+   * Convert CDC changes to Iceberg-compatible events.
+   *
+   * This is used for integration with pglake's WAL-to-Iceberg transformer.
+   */
+  async *toIcebergEvents(options: {
+    schema?: string
+    table?: string
+    shardId?: string
+  } = {}): AsyncGenerator<IcebergCDCEvent<T>, void, undefined> {
+    const schema = options.schema ?? 'public'
+    const table = options.table ?? 'unknown'
+    const shardId = options.shardId ?? 'cdc'
+
+    for await (const change of this) {
+      // Skip RESET events for Iceberg
+      if (change.__op__ === 'RESET') {
+        continue
+      }
+
+      this.lsnCounter++
+      const lsn = this.lsnCounter.toString().padStart(24, '0')
+
+      const event: IcebergCDCEvent<T> = {
+        lsn,
+        operation: change.__op__,
+        schema,
+        table,
+        changedColumns: change.__changed_columns__,
+        timestamp: Date.now(),
+        shardId,
+      }
+
+      // Extract primary key and row data
+      const rowData = { ...change } as Record<string, unknown>
+      delete rowData.__op__
+      delete rowData.__changed_columns__
+      delete rowData.__after__
+
+      const keyValue = (change as Record<string, unknown>)[this.config.key]
+      if (keyValue !== undefined) {
+        event.primaryKey = { [this.config.key]: keyValue }
+      }
+
+      if (change.__op__ === 'INSERT') {
+        event.newRow = rowData as T
+      } else if (change.__op__ === 'UPDATE') {
+        event.newRow = rowData as T
+        // For UPDATE, we could potentially track old values if needed
+      } else if (change.__op__ === 'DELETE') {
+        event.oldRow = rowData as T
+      }
+
+      yield event
+    }
+  }
+
+  // ============================================================================
+  // Private Methods
+  // ============================================================================
+
+  private handleChanges(changes: CDCChange<T>[]): void {
+    const processed = this.processChanges(changes)
+
+    for (const change of processed) {
+      // Check high water mark
+      if (this.buffer.length >= this.config.highWaterMark) {
+        // Drop oldest changes if buffer is full
+        this.buffer.shift()
+      }
+
+      this.buffer.push(change)
+
+      // If there's a waiter, resolve it immediately
+      if (this.resolveWaiter && this.buffer.length === 1) {
+        const nextChange = this.buffer.shift()
+        if (nextChange) {
+          this.resolveWaiter(nextChange)
+          this.resolveWaiter = null
+                  }
+      }
+    }
+  }
+
+  private processChanges(changes: CDCChange<T>[]): CDCChange<T>[] {
+    const result: CDCChange<T>[] = []
+
+    for (const change of changes) {
+      // Update stats
+      this.stats.totalChanges++
+      this.stats.lastChangeAt = Date.now()
+
+      switch (change.__op__) {
+        case 'INSERT':
+          this.stats.insertCount++
+          break
+        case 'UPDATE':
+          this.stats.updateCount++
+          break
+        case 'DELETE':
+          this.stats.deleteCount++
+          break
+        case 'RESET':
+          this.stats.resetCount++
+          break
+      }
+
+      // Apply filter
+      if (this.config.filter && !this.config.filter(change)) {
+        this.stats.filteredCount++
+        continue
+      }
+
+      // Apply transform
+      let processed: CDCChange<T> | null = change
+      if (this.config.transform) {
+        processed = this.config.transform(change)
+        if (!processed) {
+          this.stats.filteredCount++
+          continue
+        }
+      }
+
+      result.push(processed)
+    }
+
+    return result
+  }
+
+  private waitForChange(): Promise<CDCChange<T> | null> {
+    if (this.aborted) {
+      return Promise.resolve(null)
+    }
+
+    // If buffer has items, return immediately
+    if (this.buffer.length > 0) {
+      return Promise.resolve(this.buffer.shift() ?? null)
+    }
+
+    // Wait for next change
+    return new Promise((resolve) => {
+      this.resolveWaiter = resolve
+    })
+  }
+
+  private abort(): void {
+    this.aborted = true
+    this.stats.isActive = false
+
+    if (this.resolveWaiter) {
+      this.resolveWaiter(null)
+      this.resolveWaiter = null
+    }
+  }
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Format a CDC change as an SSE message.
+ */
+function formatSSEMessage<T>(change: CDCChange<T>, eventId: number): string {
+  const eventType = change.__op__.toLowerCase()
+  const data = JSON.stringify(change)
+  return `id: ${eventId}\nevent: ${eventType}\ndata: ${data}\n\n`
+}
+
+/**
+ * Create a LiveCDCStream from a PGLite instance.
+ *
+ * Convenience function for creating a CDC stream.
+ *
+ * @example
+ * ```typescript
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * for await (const change of cdc) {
+ *   console.log(change)
+ * }
+ * ```
+ */
+export function createLiveCDCStream<T = Record<string, unknown>>(
+  pg: PGLiteWithLive,
+  config: LiveCDCStreamConfig<T>
+): LiveCDCStream<T> {
+  return new LiveCDCStream<T>(pg, config)
+}
+
+/**
+ * Create a CDC stream that automatically converts to Iceberg events.
+ *
+ * This is a convenience wrapper for integrating with pglake's
+ * WAL-to-Iceberg transformer.
+ *
+ * @example
+ * ```typescript
+ * import { createIcebergCDCStream } from '@dotdo/postgres/streaming'
+ * import { WALToIcebergTransformer } from '@dotdo/pg-lake/ingest'
+ *
+ * const cdcEvents = createIcebergCDCStream(pg, {
+ *   query: 'SELECT * FROM orders',
+ *   key: 'id',
+ *   schema: 'public',
+ *   table: 'orders',
+ * })
+ *
+ * for await (const event of cdcEvents) {
+ *   await transformer.processEvent({
+ *     ...event,
+ *     columnTypes: { id: 'integer', ... }
+ *   })
+ * }
+ * ```
+ */
+export async function* createIcebergCDCStream<T = Record<string, unknown>>(
+  pg: PGLiteWithLive,
+  config: LiveCDCStreamConfig<T> & {
+    schema?: string
+    table?: string
+    shardId?: string
+  }
+): AsyncGenerator<IcebergCDCEvent<T>, void, undefined> {
+  const stream = new LiveCDCStream<T>(pg, config)
+  await stream.start()
+
+  const eventConfig: { schema?: string; table?: string; shardId?: string } = {}
+  if (config.schema !== undefined) {
+    eventConfig.schema = config.schema
+  }
+  if (config.table !== undefined) {
+    eventConfig.table = config.table
+  }
+  if (config.shardId !== undefined) {
+    eventConfig.shardId = config.shardId
+  }
+  yield* stream.toIcebergEvents(eventConfig)
+}
+
+/**
+ * Batch CDC changes into groups for efficient processing.
+ *
+ * @example
+ * ```typescript
+ * const batched = batchCDCChanges(cdcStream, {
+ *   maxSize: 100,
+ *   maxWaitMs: 1000,
+ * })
+ *
+ * for await (const batch of batched) {
+ *   await processChanges(batch)
+ * }
+ * ```
+ */
+export async function* batchCDCChanges<T = Record<string, unknown>>(
+  source: AsyncIterable<CDCChange<T>>,
+  options: {
+    /** Maximum batch size */
+    maxSize: number
+    /** Maximum wait time in ms before flushing partial batch */
+    maxWaitMs: number
+  }
+): AsyncGenerator<CDCChange<T>[], void, undefined> {
+  const { maxSize, maxWaitMs } = options
+  let batch: CDCChange<T>[] = []
+  let timeoutId: ReturnType<typeof setTimeout> | null = null
+  let flushPromiseResolve: (() => void) | null = null
+
+  const flush = (): CDCChange<T>[] | null => {
+    if (timeoutId) {
+      clearTimeout(timeoutId)
+      timeoutId = null
+    }
+    if (batch.length === 0) return null
+    const result = batch
+    batch = []
+    return result
+  }
+
+  const scheduleFlush = (): Promise<void> => {
+    return new Promise((resolve) => {
+      flushPromiseResolve = resolve
+      timeoutId = setTimeout(() => {
+        timeoutId = null
+        if (flushPromiseResolve) {
+          flushPromiseResolve()
+          flushPromiseResolve = null
+        }
+      }, maxWaitMs)
+    })
+  }
+
+  try {
+    for await (const change of source) {
+      batch.push(change)
+
+      if (batch.length >= maxSize) {
+        const result = flush()
+        if (result) yield result
+      } else if (batch.length === 1) {
+        // Start timeout on first item in batch
+        scheduleFlush().then(() => {
+          // Timeout triggered, but we can't yield from here
+          // The async iterator will check and flush on next iteration
+        })
+      }
+
+      // Check if timeout triggered flush
+      if (timeoutId === null && batch.length > 0) {
+        const result = flush()
+        if (result) yield result
+      }
+    }
+
+    // Flush remaining items
+    const result = flush()
+    if (result) yield result
+  } finally {
+    if (timeoutId) {
+      clearTimeout(timeoutId)
+    }
+  }
+}
+
+/**
+ * Merge multiple CDC streams into a single stream.
+ *
+ * Useful for monitoring changes across multiple tables.
+ *
+ * @example
+ * ```typescript
+ * const merged = mergeCDCStreams([
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM users', key: 'id' }),
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM orders', key: 'id' }),
+ * ])
+ *
+ * for await (const change of merged) {
+ *   console.log('Change from some table:', change)
+ * }
+ * ```
+ */
+export async function* mergeCDCStreams<T = Record<string, unknown>>(
+  streams: LiveCDCStream<T>[]
+): AsyncGenerator<CDCChange<T> & { _sourceIndex: number }, void, undefined> {
+  // Start all streams
+  await Promise.all(streams.map((s) => s.start()))
+
+  // Create merged async iterator using Promise.race
+  const iterators = streams.map((s, i) => ({
+    index: i,
+    iterator: s[Symbol.asyncIterator](),
+    done: false,
+    current: null as IteratorResult<CDCChange<T>> | null,
+  }))
+
+  // Initial fetch from all iterators
+  await Promise.all(
+    iterators.map(async (it) => {
+      it.current = await it.iterator.next()
+      it.done = it.current.done ?? false
+    })
+  )
+
+  while (iterators.some((it) => !it.done)) {
+    // Race for next available value
+    const activeIterators = iterators.filter((it) => !it.done && it.current && !it.current.done)
+
+    if (activeIterators.length === 0) break
+
+    for (const it of activeIterators) {
+      if (it.current && !it.current.done) {
+        yield { ...it.current.value, _sourceIndex: it.index }
+        it.current = await it.iterator.next()
+        it.done = it.current.done ?? false
+      }
+    }
+  }
+}