@dotdo/postgres 0.1.0

package/src/worker/cdc-backpressure.ts

@@ -0,0 +1,866 @@
+/**
+ * CDC Backpressure Coordinator
+ * Task: postgres-8pqw - CDC/WAL integration with streaming backpressure coordination
+ *
+ * Provides flow control for high-throughput CDC scenarios:
+ * - Integrates BackpressureController from streaming module
+ * - Manages producer/consumer coordination for WAL events
+ * - Tracks metrics for backpressure events
+ * - Supports multiple consumers with independent backpressure state
+ *
+ * @module worker/cdc-backpressure
+ */
+
+import {
+  BackpressureController,
+  type BackpressureConfig,
+  type BackpressureStats,
+} from '../streaming/backpressure-controller'
+import type { ChangePayload, CDCEventType } from './types'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * CDC-specific backpressure configuration
+ */
+export interface CDCBackpressureConfig extends BackpressureConfig {
+  /** Enable backpressure coordination (default: true) */
+  enabled?: boolean
+  /** Per-table high water marks (overrides global) */
+  tableWaterMarks?: Record<string, { highWaterMark: number; lowWaterMark: number }>
+  /** Sample rate for metrics collection (1 = every event, 10 = every 10th) */
+  metricsSampleRate?: number
+  /** Callback when backpressure is triggered */
+  onBackpressure?: (event: CDCBackpressureEvent) => void | Promise<void>
+  /** Callback when backpressure is relieved */
+  onRelief?: (event: CDCBackpressureEvent) => void | Promise<void>
+}
+
+/**
+ * Backpressure event for callbacks
+ */
+export interface CDCBackpressureEvent {
+  /** Event type */
+  type: 'triggered' | 'relieved'
+  /** Consumer ID that triggered the event */
+  consumerId: string
+  /** Table affected (if applicable) */
+  table?: string
+  /** Current buffer size */
+  bufferedCount: number
+  /** High water mark that was exceeded */
+  highWaterMark: number
+  /** Low water mark threshold */
+  lowWaterMark: number
+  /** Timestamp */
+  timestamp: number
+}
+
+/**
+ * Metrics for CDC backpressure operations
+ */
+export interface CDCBackpressureMetrics {
+  /** Total changes processed */
+  changesProcessed: number
+  /** Total changes dropped due to backpressure */
+  changesDropped: number
+  /** Total backpressure triggers */
+  backpressureTriggered: number
+  /** Total backpressure reliefs */
+  backpressureRelieved: number
+  /** Current buffer size across all consumers */
+  currentBufferSize: number
+  /** Peak buffer size observed */
+  peakBufferSize: number
+  /** Average processing latency (ms) */
+  avgProcessingLatencyMs: number
+  /** Number of active consumers */
+  activeConsumers: number
+  /** Per-table metrics */
+  tableMetrics: Record<string, TableBackpressureMetrics>
+  /** Time since last backpressure event (ms) */
+  timeSinceLastBackpressureMs: number
+  /** Backpressure ratio (time under backpressure / total time) */
+  backpressureRatio: number
+}
+
+/**
+ * Per-table backpressure metrics
+ */
+export interface TableBackpressureMetrics {
+  /** Changes processed for this table */
+  changesProcessed: number
+  /** Changes dropped for this table */
+  changesDropped: number
+  /** Backpressure triggers for this table */
+  backpressureTriggered: number
+  /** Current buffer size for this table */
+  currentBufferSize: number
+}
+
+/**
+ * Consumer state for tracking individual consumer backpressure
+ */
+interface ConsumerState {
+  /** Unique consumer ID */
+  id: string
+  /** Backpressure controller for this consumer */
+  controller: BackpressureController<ChangePayload>
+  /** Tables this consumer is subscribed to */
+  subscribedTables: Set<string>
+  /** Whether consumer is currently paused */
+  isPaused: boolean
+  /** Last activity timestamp */
+  lastActivity: number
+  /** Changes processed by this consumer */
+  changesProcessed: number
+}
+
+/**
+ * Change with metadata for internal tracking
+ * @internal Reserved for future use in change tracking features
+ */
+export interface _TrackedChange {
+  /** The change payload */
+  change: ChangePayload
+  /** Timestamp when change was received */
+  receivedAt: number
+  /** Timestamp when change was processed */
+  processedAt?: number
+}
+
+// ============================================================================
+// Default Configuration
+// ============================================================================
+
+const DEFAULT_CDC_BACKPRESSURE_CONFIG: Required<Omit<CDCBackpressureConfig, 'tableWaterMarks' | 'onBackpressure' | 'onRelief'>> = {
+  enabled: true,
+  highWaterMark: 1000,
+  lowWaterMark: 500,
+  maxBufferSize: 10000,
+  timeout: 30000,
+  metricsSampleRate: 1,
+}
+
+// ============================================================================
+// CDC Backpressure Coordinator
+// ============================================================================
+
+/**
+ * CDCBackpressureCoordinator manages flow control for CDC streaming.
+ *
+ * It coordinates between WAL event producers and multiple consumers (WebSocket,
+ * SSE, etc.) to prevent overwhelming slow consumers while maintaining
+ * efficient throughput for fast ones.
+ *
+ * Key features:
+ * - Per-consumer backpressure state
+ * - Per-table water marks
+ * - Comprehensive metrics tracking
+ * - Non-blocking producer (drops events for paused consumers)
+ *
+ * @example
+ * ```typescript
+ * const coordinator = new CDCBackpressureCoordinator({
+ *   highWaterMark: 1000,
+ *   lowWaterMark: 500,
+ *   onBackpressure: (event) => console.log('Backpressure triggered:', event),
+ * })
+ *
+ * // Register a consumer
+ * const consumerId = coordinator.registerConsumer(['users', 'orders'])
+ *
+ * // Produce changes from WAL
+ * await coordinator.pushChange({
+ *   table: 'users',
+ *   operation: 'INSERT',
+ *   newRow: { id: 1, name: 'Alice' },
+ * })
+ *
+ * // Consume changes (async iterator)
+ * for await (const change of coordinator.getConsumerIterator(consumerId)) {
+ *   await processChange(change)
+ * }
+ * ```
+ */
+export class CDCBackpressureCoordinator {
+  private config: Required<Omit<CDCBackpressureConfig, 'tableWaterMarks' | 'onBackpressure' | 'onRelief'>>
+  private tableWaterMarks: Record<string, { highWaterMark: number; lowWaterMark: number }>
+  private onBackpressure?: (event: CDCBackpressureEvent) => void | Promise<void>
+  private onRelief?: (event: CDCBackpressureEvent) => void | Promise<void>
+
+  private consumers: Map<string, ConsumerState> = new Map()
+  private consumerCounter = 0
+
+  // Metrics state
+  private changesProcessed = 0
+  private changesDropped = 0
+  private backpressureTriggered = 0
+  private backpressureRelieved = 0
+  private peakBufferSize = 0
+  private totalLatencyMs = 0
+  private latencySamples = 0
+  private lastBackpressureTime = 0
+  private backpressureDurationMs = 0
+  private startTime = Date.now()
+  private tableMetrics: Map<string, TableBackpressureMetrics> = new Map()
+
+  // Sample counter for metrics
+  private sampleCounter = 0
+
+  constructor(config: CDCBackpressureConfig = {}) {
+    this.config = {
+      enabled: config.enabled ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.enabled,
+      highWaterMark: config.highWaterMark ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.highWaterMark,
+      lowWaterMark: config.lowWaterMark ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.lowWaterMark,
+      maxBufferSize: config.maxBufferSize ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.maxBufferSize,
+      timeout: config.timeout ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.timeout,
+      metricsSampleRate: config.metricsSampleRate ?? DEFAULT_CDC_BACKPRESSURE_CONFIG.metricsSampleRate,
+    }
+    this.tableWaterMarks = config.tableWaterMarks ?? {}
+    if (config.onBackpressure !== undefined) {
+      this.onBackpressure = config.onBackpressure
+    }
+    if (config.onRelief !== undefined) {
+      this.onRelief = config.onRelief
+    }
+  }
+
+  // ============================================================================
+  // Consumer Management
+  // ============================================================================
+
+  /**
+   * Register a new consumer for CDC events.
+   *
+   * @param subscribedTables - Tables this consumer is interested in (empty = all)
+   * @param config - Optional per-consumer backpressure config
+   * @returns Consumer ID for subsequent operations
+   */
+  registerConsumer(
+    subscribedTables: string[] = [],
+    config?: Partial<BackpressureConfig>
+  ): string {
+    const consumerId = `cdc-consumer-${Date.now()}-${++this.consumerCounter}`
+
+    // Determine water marks (per-consumer > per-table > global)
+    const highWaterMark = config?.highWaterMark ?? this.config.highWaterMark
+    const lowWaterMark = config?.lowWaterMark ?? this.config.lowWaterMark
+
+    const controller = new BackpressureController<ChangePayload>({
+      highWaterMark,
+      lowWaterMark,
+      maxBufferSize: config?.maxBufferSize ?? this.config.maxBufferSize,
+      timeout: config?.timeout ?? this.config.timeout,
+    })
+
+    const state: ConsumerState = {
+      id: consumerId,
+      controller,
+      subscribedTables: new Set(subscribedTables),
+      isPaused: false,
+      lastActivity: Date.now(),
+      changesProcessed: 0,
+    }
+
+    this.consumers.set(consumerId, state)
+
+    return consumerId
+  }
+
+  /**
+   * Unregister a consumer and clean up resources.
+   *
+   * @param consumerId - Consumer to unregister
+   * @returns True if consumer was found and removed
+   */
+  unregisterConsumer(consumerId: string): boolean {
+    const state = this.consumers.get(consumerId)
+    if (!state) {
+      return false
+    }
+
+    // End the controller to signal completion
+    state.controller.end()
+    this.consumers.delete(consumerId)
+
+    return true
+  }
+
+  /**
+   * Get async iterator for consuming changes.
+   *
+   * @param consumerId - Consumer ID
+   * @returns Async iterable of changes
+   */
+  getConsumerIterator(consumerId: string): AsyncIterable<ChangePayload> {
+    const state = this.consumers.get(consumerId)
+    if (!state) {
+      throw new Error(`Consumer ${consumerId} not found`)
+    }
+
+    return state.controller
+  }
+
+  /**
+   * Check if a consumer is registered.
+   */
+  hasConsumer(consumerId: string): boolean {
+    return this.consumers.has(consumerId)
+  }
+
+  /**
+   * Get all registered consumer IDs.
+   */
+  getConsumerIds(): string[] {
+    return Array.from(this.consumers.keys())
+  }
+
+  /**
+   * Get consumer count.
+   */
+  getConsumerCount(): number {
+    return this.consumers.size
+  }
+
+  // ============================================================================
+  // Change Production
+  // ============================================================================
+
+  /**
+   * Push a change to all interested consumers.
+   *
+   * This method is non-blocking for the producer - if a consumer is paused
+   * due to backpressure, the change is dropped for that consumer and metrics
+   * are updated accordingly.
+   *
+   * @param change - The change payload to push
+   * @returns Number of consumers that received the change
+   */
+  async pushChange(change: ChangePayload): Promise<number> {
+    if (!this.config.enabled) {
+      return 0
+    }
+
+    const receivedAt = Date.now()
+    let deliveredCount = 0
+
+    // Initialize table metrics if needed
+    if (!this.tableMetrics.has(change.table)) {
+      this.tableMetrics.set(change.table, {
+        changesProcessed: 0,
+        changesDropped: 0,
+        backpressureTriggered: 0,
+        currentBufferSize: 0,
+      })
+    }
+
+    for (const [consumerId, state] of this.consumers.entries()) {
+      // Check if consumer is interested in this table
+      if (state.subscribedTables.size > 0 && !state.subscribedTables.has(change.table)) {
+        continue
+      }
+
+      // Get table-specific water marks if available
+      const tableConfig = this.tableWaterMarks[change.table]
+      const stats = state.controller.getStats()
+
+      // Check if consumer is paused
+      const wasPaused = state.isPaused
+
+      // Try to push the change
+      try {
+        // Check if we would trigger backpressure
+        const effectiveHighWaterMark = tableConfig?.highWaterMark ?? this.config.highWaterMark
+
+        if (stats.bufferedCount >= effectiveHighWaterMark) {
+          // Consumer is overwhelmed - drop the change
+          this.changesDropped++
+          this.updateTableMetrics(change.table, 'dropped')
+
+          // Trigger backpressure callback if transitioning
+          if (!wasPaused) {
+            state.isPaused = true
+            this.backpressureTriggered++
+            this.lastBackpressureTime = Date.now()
+
+            this.updateTableMetrics(change.table, 'backpressure')
+
+            if (this.onBackpressure) {
+              await this.onBackpressure({
+                type: 'triggered',
+                consumerId,
+                table: change.table,
+                bufferedCount: stats.bufferedCount,
+                highWaterMark: effectiveHighWaterMark,
+                lowWaterMark: tableConfig?.lowWaterMark ?? this.config.lowWaterMark,
+                timestamp: Date.now(),
+              })
+            }
+          }
+
+          continue
+        }
+
+        // Push the change (non-blocking since we checked water marks)
+        await state.controller.push(change)
+        deliveredCount++
+        state.lastActivity = Date.now()
+        state.changesProcessed++
+
+        // Track metrics
+        this.changesProcessed++
+        this.updateTableMetrics(change.table, 'processed')
+
+        // Track latency (sampled)
+        this.sampleCounter++
+        if (this.sampleCounter % this.config.metricsSampleRate === 0) {
+          const latency = Date.now() - receivedAt
+          this.totalLatencyMs += latency
+          this.latencySamples++
+        }
+
+        // Check if we should relieve backpressure
+        const effectiveLowWaterMark = tableConfig?.lowWaterMark ?? this.config.lowWaterMark
+        const newStats = state.controller.getStats()
+
+        if (wasPaused && newStats.bufferedCount <= effectiveLowWaterMark) {
+          state.isPaused = false
+          this.backpressureRelieved++
+
+          // Track backpressure duration
+          if (this.lastBackpressureTime > 0) {
+            this.backpressureDurationMs += Date.now() - this.lastBackpressureTime
+          }
+
+          if (this.onRelief) {
+            await this.onRelief({
+              type: 'relieved',
+              consumerId,
+              table: change.table,
+              bufferedCount: newStats.bufferedCount,
+              highWaterMark: effectiveHighWaterMark,
+              lowWaterMark: effectiveLowWaterMark,
+              timestamp: Date.now(),
+            })
+          }
+        }
+
+        // Update peak buffer size
+        if (newStats.bufferedCount > this.peakBufferSize) {
+          this.peakBufferSize = newStats.bufferedCount
+        }
+
+      } catch (error) {
+        // Buffer overflow or other error - drop the change
+        this.changesDropped++
+        this.updateTableMetrics(change.table, 'dropped')
+      }
+    }
+
+    return deliveredCount
+  }
+
+  /**
+   * Push multiple changes in batch.
+   *
+   * @param changes - Array of change payloads
+   * @returns Total number of deliveries (sum across all consumers)
+   */
+  async pushChanges(changes: ChangePayload[]): Promise<number> {
+    let totalDelivered = 0
+    for (const change of changes) {
+      totalDelivered += await this.pushChange(change)
+    }
+    return totalDelivered
+  }
+
+  // ============================================================================
+  // Consumer Control
+  // ============================================================================
+
+  /**
+   * Manually pause a consumer (producer will drop changes for this consumer).
+   *
+   * @param consumerId - Consumer to pause
+   * @returns True if consumer was found
+   */
+  pauseConsumer(consumerId: string): boolean {
+    const state = this.consumers.get(consumerId)
+    if (!state) {
+      return false
+    }
+
+    if (!state.isPaused) {
+      state.isPaused = true
+      state.controller.pause()
+      this.backpressureTriggered++
+      this.lastBackpressureTime = Date.now()
+    }
+
+    return true
+  }
+
+  /**
+   * Resume a paused consumer.
+   *
+   * @param consumerId - Consumer to resume
+   * @returns True if consumer was found
+   */
+  resumeConsumer(consumerId: string): boolean {
+    const state = this.consumers.get(consumerId)
+    if (!state) {
+      return false
+    }
+
+    if (state.isPaused) {
+      state.isPaused = false
+      state.controller.resume()
+      this.backpressureRelieved++
+
+      if (this.lastBackpressureTime > 0) {
+        this.backpressureDurationMs += Date.now() - this.lastBackpressureTime
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Check if a consumer is paused.
+   */
+  isConsumerPaused(consumerId: string): boolean {
+    const state = this.consumers.get(consumerId)
+    return state?.isPaused ?? false
+  }
+
+  /**
+   * Get stats for a specific consumer.
+   */
+  getConsumerStats(consumerId: string): BackpressureStats | null {
+    const state = this.consumers.get(consumerId)
+    if (!state) {
+      return null
+    }
+    return state.controller.getStats()
+  }
+
+  // ============================================================================
+  // Metrics
+  // ============================================================================
+
+  /**
+   * Get comprehensive metrics for CDC backpressure.
+   */
+  getMetrics(): CDCBackpressureMetrics {
+    const now = Date.now()
+    const totalTime = now - this.startTime
+
+    // Calculate current buffer size across all consumers
+    let currentBufferSize = 0
+    for (const state of this.consumers.values()) {
+      currentBufferSize += state.controller.getStats().bufferedCount
+    }
+
+    // Calculate backpressure ratio
+    const backpressureRatio = totalTime > 0
+      ? this.backpressureDurationMs / totalTime
+      : 0
+
+    // Calculate average latency
+    const avgProcessingLatencyMs = this.latencySamples > 0
+      ? this.totalLatencyMs / this.latencySamples
+      : 0
+
+    // Time since last backpressure
+    const timeSinceLastBackpressureMs = this.lastBackpressureTime > 0
+      ? now - this.lastBackpressureTime
+      : totalTime
+
+    // Convert table metrics to plain object
+    const tableMetrics: Record<string, TableBackpressureMetrics> = {}
+    for (const [table, metrics] of this.tableMetrics.entries()) {
+      tableMetrics[table] = { ...metrics }
+    }
+
+    return {
+      changesProcessed: this.changesProcessed,
+      changesDropped: this.changesDropped,
+      backpressureTriggered: this.backpressureTriggered,
+      backpressureRelieved: this.backpressureRelieved,
+      currentBufferSize,
+      peakBufferSize: this.peakBufferSize,
+      avgProcessingLatencyMs,
+      activeConsumers: this.consumers.size,
+      tableMetrics,
+      timeSinceLastBackpressureMs,
+      backpressureRatio,
+    }
+  }
+
+  /**
+   * Reset metrics counters.
+   */
+  resetMetrics(): void {
+    this.changesProcessed = 0
+    this.changesDropped = 0
+    this.backpressureTriggered = 0
+    this.backpressureRelieved = 0
+    this.peakBufferSize = 0
+    this.totalLatencyMs = 0
+    this.latencySamples = 0
+    this.lastBackpressureTime = 0
+    this.backpressureDurationMs = 0
+    this.startTime = Date.now()
+    this.tableMetrics.clear()
+    this.sampleCounter = 0
+
+    // Reset per-consumer stats
+    for (const state of this.consumers.values()) {
+      state.controller.resetStats()
+      state.changesProcessed = 0
+    }
+  }
+
+  /**
+   * Update table-specific metrics.
+   */
+  private updateTableMetrics(
+    table: string,
+    event: 'processed' | 'dropped' | 'backpressure'
+  ): void {
+    const metrics = this.tableMetrics.get(table)
+    if (!metrics) return
+
+    switch (event) {
+      case 'processed':
+        metrics.changesProcessed++
+        break
+      case 'dropped':
+        metrics.changesDropped++
+        break
+      case 'backpressure':
+        metrics.backpressureTriggered++
+        break
+    }
+  }
+
+  // ============================================================================
+  // Configuration
+  // ============================================================================
+
+  /**
+   * Check if backpressure coordination is enabled.
+   */
+  isEnabled(): boolean {
+    return this.config.enabled
+  }
+
+  /**
+   * Enable backpressure coordination.
+   */
+  enable(): void {
+    this.config.enabled = true
+  }
+
+  /**
+   * Disable backpressure coordination.
+   */
+  disable(): void {
+    this.config.enabled = false
+  }
+
+  /**
+   * Update configuration.
+   */
+  updateConfig(config: Partial<CDCBackpressureConfig>): void {
+    if (config.enabled !== undefined) {
+      this.config.enabled = config.enabled
+    }
+    if (config.highWaterMark !== undefined) {
+      this.config.highWaterMark = config.highWaterMark
+    }
+    if (config.lowWaterMark !== undefined) {
+      this.config.lowWaterMark = config.lowWaterMark
+    }
+    if (config.maxBufferSize !== undefined) {
+      this.config.maxBufferSize = config.maxBufferSize
+    }
+    if (config.timeout !== undefined) {
+      this.config.timeout = config.timeout
+    }
+    if (config.metricsSampleRate !== undefined) {
+      this.config.metricsSampleRate = config.metricsSampleRate
+    }
+    if (config.tableWaterMarks !== undefined) {
+      this.tableWaterMarks = { ...this.tableWaterMarks, ...config.tableWaterMarks }
+    }
+    if (config.onBackpressure !== undefined) {
+      this.onBackpressure = config.onBackpressure
+    }
+    if (config.onRelief !== undefined) {
+      this.onRelief = config.onRelief
+    }
+  }
+
+  /**
+   * Set table-specific water marks.
+   */
+  setTableWaterMarks(
+    table: string,
+    waterMarks: { highWaterMark: number; lowWaterMark: number }
+  ): void {
+    this.tableWaterMarks[table] = waterMarks
+  }
+
+  /**
+   * Get current configuration.
+   */
+  getConfig(): CDCBackpressureConfig {
+    const config: CDCBackpressureConfig = {
+      ...this.config,
+      tableWaterMarks: { ...this.tableWaterMarks },
+    }
+    if (this.onBackpressure !== undefined) {
+      config.onBackpressure = this.onBackpressure
+    }
+    if (this.onRelief !== undefined) {
+      config.onRelief = this.onRelief
+    }
+    return config
+  }
+
+  // ============================================================================
+  // Cleanup
+  // ============================================================================
+
+  /**
+   * Shutdown the coordinator and clean up all resources.
+   */
+  shutdown(): void {
+    for (const state of this.consumers.values()) {
+      state.controller.end()
+    }
+    this.consumers.clear()
+  }
+
+  /**
+   * Abort all consumers with an error.
+   */
+  abort(error?: Error): void {
+    for (const state of this.consumers.values()) {
+      state.controller.abort(error)
+    }
+    this.consumers.clear()
+  }
+}
+
+// ============================================================================
+// Factory Function
+// ============================================================================
+
+/**
+ * Create a new CDC backpressure coordinator.
+ *
+ * @param config - Optional configuration
+ * @returns New CDCBackpressureCoordinator instance
+ */
+export function createCDCBackpressureCoordinator(
+  config?: CDCBackpressureConfig
+): CDCBackpressureCoordinator {
+  return new CDCBackpressureCoordinator(config)
+}
+
+// ============================================================================
+// Integration Helpers
+// ============================================================================
+
+/**
+ * Create a backpressure-aware change handler for CDCManager.
+ *
+ * This function creates a wrapper that integrates backpressure coordination
+ * with the existing CDCManager.broadcastChange method.
+ *
+ * @param coordinator - The backpressure coordinator
+ * @param originalBroadcast - Original broadcast function
+ * @returns Wrapped broadcast function with backpressure support
+ */
+export function createBackpressureAwareBroadcast(
+  coordinator: CDCBackpressureCoordinator,
+  originalBroadcast: (change: ChangePayload) => void | Promise<void>
+): (change: ChangePayload) => Promise<void> {
+  return async (change: ChangePayload) => {
+    // Push to coordinator for consumers that support backpressure
+    await coordinator.pushChange(change)
+
+    // Also call original broadcast for non-backpressure-aware consumers
+    await originalBroadcast(change)
+  }
+}
+
+/**
+ * WAL event types for CDC streaming
+ */
+export type WALEventType = 'INSERT' | 'UPDATE' | 'DELETE' | 'TRUNCATE' | 'BEGIN' | 'COMMIT'
+
+/**
+ * WAL event structure
+ */
+export interface WALEvent {
+  /** Event type */
+  type: WALEventType
+  /** Table name (for data events) */
+  table?: string
+  /** Schema name */
+  schema?: string
+  /** Log Sequence Number */
+  lsn: string
+  /** Transaction ID */
+  xid?: number
+  /** New row data */
+  newRow?: Record<string, unknown>
+  /** Old row data */
+  oldRow?: Record<string, unknown>
+  /** Timestamp */
+  timestamp: number
+}
+
+/**
+ * Convert WAL event to ChangePayload for CDC.
+ */
+export function walEventToChangePayload(event: WALEvent): ChangePayload | null {
+  // Only data events are relevant for CDC
+  if (!event.table || !['INSERT', 'UPDATE', 'DELETE'].includes(event.type)) {
+    return null
+  }
+
+  return {
+    table: event.table,
+    operation: event.type as CDCEventType,
+    newRow: event.newRow,
+    oldRow: event.oldRow,
+    lsn: event.lsn,
+    timestamp: event.timestamp,
+  }
+}
+
+/**
+ * Create a WAL event stream consumer with backpressure.
+ *
+ * @param coordinator - The backpressure coordinator
+ * @param walStream - Source WAL event stream
+ * @returns Consumer ID that can be used for cleanup
+ */
+export async function consumeWALStream(
+  coordinator: CDCBackpressureCoordinator,
+  walStream: AsyncIterable<WALEvent>
+): Promise<void> {
+  for await (const event of walStream) {
+    const change = walEventToChangePayload(event)
+    if (change) {
+      await coordinator.pushChange(change)
+    }
+  }
+}