@dotdo/postgres 0.1.0

package/src/streaming/cdc-iceberg-connector.ts

@@ -0,0 +1,605 @@
+/**
+ * CDC to Iceberg Connector
+ *
+ * Bridges LiveCDCStream with pglake's WAL-to-Iceberg transformer.
+ * Converts PGLite live.changes() events into the format expected by
+ * the Iceberg ingestion pipeline.
+ *
+ * ## Architecture
+ *
+ * ```
+ * ┌─────────────────┐    ┌──────────────────┐    ┌───────────────────┐
+ * │ PGLite DO       │───>│ LiveCDCStream    │───>│ CDCIcebergBridge  │
+ * │ (live.changes)  │    │ (async iterator) │    │ (transformation)  │
+ * └─────────────────┘    └──────────────────┘    └─────────┬─────────┘
+ *                                                          │
+ *                                                          v
+ *                        ┌───────────────────┐    ┌───────────────────┐
+ *                        │ Parquet Files     │<───│ WALToIceberg      │
+ *                        │ (R2 Storage)      │    │ Transformer       │
+ *                        └───────────────────┘    └───────────────────┘
+ * ```
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { CDCIcebergBridge } from '@dotdo/postgres/streaming'
+ * import { WALToIcebergTransformer } from '@dotdo/pg-lake/ingest'
+ *
+ * const bridge = new CDCIcebergBridge({
+ *   pg,
+ *   transformer,
+ *   tables: [
+ *     { query: 'SELECT * FROM users', key: 'id', schema: 'public', table: 'users' },
+ *     { query: 'SELECT * FROM orders', key: 'id', schema: 'public', table: 'orders' },
+ *   ],
+ * })
+ *
+ * await bridge.start()
+ * ```
+ *
+ * @module streaming/cdc-iceberg-connector
+ */
+
+import {
+  LiveCDCStream,
+  type PGLiteWithLive,
+  type CDCChange,
+} from './live-cdc-stream'
+
+// ============================================================================
+// Type Definitions
+// ============================================================================
+
+/**
+ * WAL event data format expected by WALToIcebergTransformer
+ */
+export interface WALEventData {
+  operation: 'INSERT' | 'UPDATE' | 'DELETE' | 'TRUNCATE' | 'DDL'
+  schema: string
+  table: string
+  primaryKey?: Record<string, unknown>
+  newRow?: Record<string, unknown>
+  oldRow?: Record<string, unknown>
+  columnTypes?: Record<string, string>
+}
+
+/**
+ * WAL entry format for batch processing
+ */
+export interface WALEntry {
+  lsn: string
+  segmentFile: string
+  offset: number
+  xid: string
+  commitTimestamp: number
+  events: WALEventData[]
+  sizeBytes: number
+}
+
+/**
+ * WAL-to-Iceberg transformer interface (minimal interface for integration)
+ */
+export interface IcebergTransformer {
+  initialize(): Promise<void>
+  processWALEntry(entry: WALEntry): Promise<TransformationResult>
+  processEvent(event: WALEventData & { lsn: string; timestamp: number; shardId: string }): Promise<void>
+  flush(): Promise<TransformationResult>
+  shutdown(): Promise<void>
+}
+
+/**
+ * Transformation result from Iceberg transformer
+ */
+export interface TransformationResult {
+  eventsProcessed: number
+  filesWritten: unknown[]
+  manifestsCreated: unknown[]
+  snapshot?: unknown
+  durationMs: number
+  errors: string[]
+}
+
+/**
+ * Table configuration for CDC streaming
+ */
+export interface CDCTableConfig {
+  /** SQL query to watch */
+  query: string
+  /** Primary key column */
+  key: string
+  /** PostgreSQL schema name */
+  schema: string
+  /** Table name */
+  table: string
+  /** Column types for schema inference (optional) */
+  columnTypes?: Record<string, string>
+  /** Optional filter function */
+  filter?: (change: CDCChange<Record<string, unknown>>) => boolean
+}
+
+/**
+ * CDC-to-Iceberg bridge configuration
+ */
+export interface CDCIcebergBridgeConfig {
+  /** PGLite instance with live extension */
+  pg: PGLiteWithLive
+  /** WAL-to-Iceberg transformer */
+  transformer: IcebergTransformer
+  /** Tables to stream */
+  tables: CDCTableConfig[]
+  /** Shard ID for this DO instance */
+  shardId?: string
+  /** Batch size before flushing to Iceberg (default: 1000) */
+  batchSize?: number
+  /** Flush interval in ms (default: 5000) */
+  flushIntervalMs?: number
+  /** AbortSignal for cancellation */
+  signal?: AbortSignal
+  /** Error handler */
+  onError?: (error: Error, table: string) => void
+  /** Metrics handler */
+  onMetrics?: (metrics: CDCBridgeMetrics) => void
+}
+
+/**
+ * Bridge metrics
+ */
+export interface CDCBridgeMetrics {
+  /** Total events processed across all tables */
+  totalEventsProcessed: number
+  /** Events by table */
+  eventsByTable: Map<string, number>
+  /** Total batches flushed */
+  batchesFlushed: number
+  /** Total errors encountered */
+  errorsCount: number
+  /** Bridge start time */
+  startedAt: number
+  /** Last flush time */
+  lastFlushAt: number | null
+  /** Events per second (rolling average) */
+  eventsPerSecond: number
+  /** Active streams count */
+  activeStreams: number
+}
+
+// ============================================================================
+// Default Configuration
+// ============================================================================
+
+const DEFAULT_BATCH_SIZE = 1000
+const DEFAULT_FLUSH_INTERVAL_MS = 5000
+
+// ============================================================================
+// CDCIcebergBridge Class
+// ============================================================================
+
+/**
+ * CDCIcebergBridge connects PGLite CDC streams to Iceberg ingestion.
+ *
+ * This class manages multiple CDC streams (one per table) and routes
+ * changes to a WAL-to-Iceberg transformer for persistence.
+ *
+ * ## Lifecycle
+ *
+ * 1. Create bridge with PGLite instance and transformer
+ * 2. Call start() to begin streaming
+ * 3. Changes flow automatically to Iceberg
+ * 4. Call stop() to gracefully shutdown
+ *
+ * @example Basic usage
+ * ```typescript
+ * const bridge = new CDCIcebergBridge({
+ *   pg,
+ *   transformer,
+ *   tables: [
+ *     { query: 'SELECT * FROM users', key: 'id', schema: 'public', table: 'users' },
+ *   ],
+ * })
+ *
+ * await bridge.start()
+ *
+ * // Bridge runs continuously, processing changes as they occur
+ * // Call stop() when done:
+ * await bridge.stop()
+ * ```
+ */
+export class CDCIcebergBridge {
+  private readonly pg: PGLiteWithLive
+  private readonly transformer: IcebergTransformer
+  private readonly tables: CDCTableConfig[]
+  private readonly config: {
+    shardId: string
+    batchSize: number
+    flushIntervalMs: number
+    signal?: AbortSignal
+    onError?: (error: Error, table: string) => void
+    onMetrics?: (metrics: CDCBridgeMetrics) => void
+  }
+
+  private streams: Map<string, LiveCDCStream<Record<string, unknown>>> = new Map()
+  private eventBuffer: Array<WALEventData & { lsn: string; timestamp: number; shardId: string }> = []
+  private lsnCounter = 0
+  private flushTimer: ReturnType<typeof setInterval> | null = null
+  private started = false
+  private stopping = false
+
+  private metrics: CDCBridgeMetrics = {
+    totalEventsProcessed: 0,
+    eventsByTable: new Map(),
+    batchesFlushed: 0,
+    errorsCount: 0,
+    startedAt: 0,
+    lastFlushAt: null,
+    eventsPerSecond: 0,
+    activeStreams: 0,
+  }
+
+  constructor(config: CDCIcebergBridgeConfig) {
+    this.pg = config.pg
+    this.transformer = config.transformer
+    this.tables = config.tables
+    this.config = {
+      shardId: config.shardId ?? `do-${Date.now()}`,
+      batchSize: config.batchSize ?? DEFAULT_BATCH_SIZE,
+      flushIntervalMs: config.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS,
+      ...(config.signal !== undefined && { signal: config.signal }),
+      ...(config.onError !== undefined && { onError: config.onError }),
+      ...(config.onMetrics !== undefined && { onMetrics: config.onMetrics }),
+    }
+
+    // Handle abort signal
+    if (this.config.signal) {
+      this.config.signal.addEventListener('abort', () => {
+        this.stop().catch(console.error)
+      }, { once: true })
+    }
+  }
+
+  /**
+   * Start the CDC-to-Iceberg bridge.
+   *
+   * Creates CDC streams for all configured tables and starts
+   * processing changes.
+   */
+  async start(): Promise<void> {
+    if (this.started) {
+      throw new Error('CDCIcebergBridge already started')
+    }
+
+    this.started = true
+    this.metrics.startedAt = Date.now()
+
+    // Initialize transformer
+    await this.transformer.initialize()
+
+    // Create CDC streams for each table
+    for (const tableConfig of this.tables) {
+      const tableName = `${tableConfig.schema}.${tableConfig.table}`
+
+      const stream = new LiveCDCStream<Record<string, unknown>>(this.pg, {
+        query: tableConfig.query,
+        key: tableConfig.key,
+        ...(tableConfig.filter !== undefined && { filter: tableConfig.filter }),
+        ...(this.config.signal !== undefined && { signal: this.config.signal }),
+      })
+
+      this.streams.set(tableName, stream)
+      this.metrics.eventsByTable.set(tableName, 0)
+
+      // Start consuming the stream
+      this.consumeStream(stream, tableConfig).catch((error) => {
+        this.handleError(error, tableName)
+      })
+    }
+
+    this.metrics.activeStreams = this.streams.size
+
+    // Start periodic flush timer
+    this.flushTimer = setInterval(() => {
+      if (this.eventBuffer.length > 0) {
+        this.flush().catch((error) => {
+          this.handleError(error, 'flush')
+        })
+      }
+    }, this.config.flushIntervalMs)
+
+    // Report initial metrics
+    this.reportMetrics()
+  }
+
+  /**
+   * Stop the bridge and flush remaining events.
+   */
+  async stop(): Promise<void> {
+    if (!this.started || this.stopping) return
+
+    this.stopping = true
+
+    // Stop flush timer
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer)
+      this.flushTimer = null
+    }
+
+    // Stop all streams
+    const stopPromises = Array.from(this.streams.values()).map((stream) => stream.stop())
+    await Promise.all(stopPromises)
+
+    this.streams.clear()
+    this.metrics.activeStreams = 0
+
+    // Flush remaining events
+    if (this.eventBuffer.length > 0) {
+      await this.flush()
+    }
+
+    // Shutdown transformer
+    await this.transformer.shutdown()
+
+    this.started = false
+    this.stopping = false
+  }
+
+  /**
+   * Manually trigger a flush of buffered events.
+   */
+  async flush(): Promise<TransformationResult> {
+    if (this.eventBuffer.length === 0) {
+      return {
+        eventsProcessed: 0,
+        filesWritten: [],
+        manifestsCreated: [],
+        durationMs: 0,
+        errors: [],
+      }
+    }
+
+    // Clear buffer before flush
+    this.eventBuffer = []
+
+    const result = await this.transformer.flush()
+
+    this.metrics.batchesFlushed++
+    this.metrics.lastFlushAt = Date.now()
+    this.updateEventsPerSecond()
+    this.reportMetrics()
+
+    return result
+  }
+
+  /**
+   * Get current bridge metrics.
+   */
+  getMetrics(): CDCBridgeMetrics {
+    return { ...this.metrics, eventsByTable: new Map(this.metrics.eventsByTable) }
+  }
+
+  /**
+   * Check if bridge is currently running.
+   */
+  get isRunning(): boolean {
+    return this.started && !this.stopping
+  }
+
+  // ============================================================================
+  // Private Methods
+  // ============================================================================
+
+  private async consumeStream(
+    stream: LiveCDCStream<Record<string, unknown>>,
+    tableConfig: CDCTableConfig
+  ): Promise<void> {
+    const tableName = `${tableConfig.schema}.${tableConfig.table}`
+
+    try {
+      await stream.start()
+
+      for await (const change of stream) {
+        if (this.stopping) break
+
+        // Skip RESET events
+        if (change.__op__ === 'RESET') continue
+
+        // Convert to WAL event
+        const walEvent = this.convertToWALEvent(change, tableConfig)
+
+        // Add to buffer
+        this.eventBuffer.push(walEvent)
+
+        // Update metrics
+        this.metrics.totalEventsProcessed++
+        const tableCount = this.metrics.eventsByTable.get(tableName) ?? 0
+        this.metrics.eventsByTable.set(tableName, tableCount + 1)
+
+        // Process event directly with transformer
+        await this.transformer.processEvent(walEvent)
+
+        // Check batch size for flush
+        if (this.eventBuffer.length >= this.config.batchSize) {
+          await this.flush()
+        }
+      }
+    } catch (error) {
+      this.handleError(error as Error, tableName)
+    }
+  }
+
+  private convertToWALEvent(
+    change: CDCChange<Record<string, unknown>>,
+    tableConfig: CDCTableConfig
+  ): WALEventData & { lsn: string; timestamp: number; shardId: string } {
+    this.lsnCounter++
+    const lsn = this.lsnCounter.toString().padStart(24, '0')
+
+    // Extract row data (remove CDC metadata)
+    const rowData = { ...change } as Record<string, unknown>
+    delete rowData.__op__
+    delete rowData.__changed_columns__
+    delete rowData.__after__
+
+    const event: WALEventData & { lsn: string; timestamp: number; shardId: string } = {
+      lsn,
+      timestamp: Date.now(),
+      shardId: this.config.shardId,
+      operation: change.__op__ as 'INSERT' | 'UPDATE' | 'DELETE',
+      schema: tableConfig.schema,
+      table: tableConfig.table,
+      ...(tableConfig.columnTypes !== undefined && { columnTypes: tableConfig.columnTypes }),
+    }
+
+    // Extract primary key
+    const keyValue = rowData[tableConfig.key]
+    if (keyValue !== undefined) {
+      event.primaryKey = { [tableConfig.key]: keyValue }
+    }
+
+    // Set row data based on operation
+    if (change.__op__ === 'INSERT') {
+      event.newRow = rowData
+    } else if (change.__op__ === 'UPDATE') {
+      event.newRow = rowData
+      // Note: For updates, we only have the new values
+      // If we need old values, we'd need to track them separately
+    } else if (change.__op__ === 'DELETE') {
+      event.oldRow = rowData
+    }
+
+    return event
+  }
+
+  private handleError(error: Error, context: string): void {
+    this.metrics.errorsCount++
+
+    if (this.config.onError) {
+      this.config.onError(error, context)
+    } else {
+      console.error(`[CDCIcebergBridge] Error in ${context}:`, error)
+    }
+  }
+
+  private updateEventsPerSecond(): void {
+    if (this.metrics.startedAt > 0) {
+      const elapsedSeconds = (Date.now() - this.metrics.startedAt) / 1000
+      if (elapsedSeconds > 0) {
+        this.metrics.eventsPerSecond = this.metrics.totalEventsProcessed / elapsedSeconds
+      }
+    }
+  }
+
+  private reportMetrics(): void {
+    if (this.config.onMetrics) {
+      this.config.onMetrics(this.getMetrics())
+    }
+  }
+}
+
+// ============================================================================
+// Factory Functions
+// ============================================================================
+
+/**
+ * Create a CDC-to-Iceberg bridge for a single table.
+ *
+ * Convenience function for single-table streaming scenarios.
+ *
+ * @example
+ * ```typescript
+ * const bridge = await createSingleTableBridge(pg, transformer, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ *   schema: 'public',
+ *   table: 'users',
+ * })
+ *
+ * await bridge.start()
+ * ```
+ */
+export function createSingleTableBridge(
+  pg: PGLiteWithLive,
+  transformer: IcebergTransformer,
+  tableConfig: CDCTableConfig,
+  options?: Omit<CDCIcebergBridgeConfig, 'pg' | 'transformer' | 'tables'>
+): CDCIcebergBridge {
+  return new CDCIcebergBridge({
+    pg,
+    transformer,
+    tables: [tableConfig],
+    ...options,
+  })
+}
+
+/**
+ * Create a simple CDC stream that outputs Iceberg-compatible events.
+ *
+ * This is a lighter-weight alternative when you don't need the full
+ * WAL-to-Iceberg transformer integration.
+ *
+ * @example
+ * ```typescript
+ * const events = createSimpleCDCIcebergStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ *   schema: 'public',
+ *   table: 'users',
+ * })
+ *
+ * for await (const event of events) {
+ *   // Process event manually
+ *   console.log(event.lsn, event.operation, event.newRow)
+ * }
+ * ```
+ */
+export async function* createSimpleCDCIcebergStream(
+  pg: PGLiteWithLive,
+  config: CDCTableConfig & { shardId?: string }
+): AsyncGenerator<WALEventData & { lsn: string; timestamp: number; shardId: string }> {
+  const stream = new LiveCDCStream<Record<string, unknown>>(pg, {
+    query: config.query,
+    key: config.key,
+    ...(config.filter !== undefined && { filter: config.filter }),
+  })
+
+  const shardId = config.shardId ?? `do-${Date.now()}`
+  let lsnCounter = 0
+
+  await stream.start()
+
+  for await (const change of stream) {
+    if (change.__op__ === 'RESET') continue
+
+    lsnCounter++
+    const lsn = lsnCounter.toString().padStart(24, '0')
+
+    const rowData = { ...change } as Record<string, unknown>
+    delete rowData.__op__
+    delete rowData.__changed_columns__
+    delete rowData.__after__
+
+    const event: WALEventData & { lsn: string; timestamp: number; shardId: string } = {
+      lsn,
+      timestamp: Date.now(),
+      shardId,
+      operation: change.__op__ as 'INSERT' | 'UPDATE' | 'DELETE',
+      schema: config.schema,
+      table: config.table,
+      ...(config.columnTypes !== undefined && { columnTypes: config.columnTypes }),
+    }
+
+    const keyValue = rowData[config.key]
+    if (keyValue !== undefined) {
+      event.primaryKey = { [config.key]: keyValue }
+    }
+
+    if (change.__op__ === 'INSERT') {
+      event.newRow = rowData
+    } else if (change.__op__ === 'UPDATE') {
+      event.newRow = rowData
+    } else if (change.__op__ === 'DELETE') {
+      event.oldRow = rowData
+    }
+
+    yield event
+  }
+}