@dotdo/postgres 0.1.0

package/src/iceberg/transformer.ts

@@ -0,0 +1,1029 @@
+/**
+ * WAL to Iceberg Transformer
+ * Task: postgres-5iv - WAL to Iceberg transformation
+ *
+ * Transforms streaming WAL data into Iceberg table format for efficient querying.
+ * Implements:
+ * - Continuous transformation pipeline
+ * - Batched micro-batch processing
+ * - Schema mapping from WAL to Iceberg
+ * - Deduplication of replayed/retried WAL
+ * - Exactly-once processing semantics via checkpointing
+ */
+
+import type { SerializedWALEntry } from '../worker/wal'
+import type { WALSegmentMetadata } from '../worker/wal-r2'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Re-export for external use
+export type { R2WALStorage } from '../worker/wal-r2'
+import type {
+  IcebergTableMetadata,
+  IcebergSnapshot,
+  IcebergSchema,
+  IcebergPartitionSpec,
+} from './types'
+import { IcebergWALWriter, createIcebergWALWriter } from './writer'
+import { walEntryToParquetRecord, type WALParquetRecord } from './parquet'
+import { createWALSchema, createWALPartitionSpec } from './schema'
+
+// Re-export schema utils for external use
+export { getPartitionValue, generatePartitionPath } from './schema'
+
+// Module-level logger for transformer errors
+const transformerLogger: ILogger = createLogger({
+  level: LogLevel.ERROR,
+  prefix: '[WALIcebergTransformer]',
+})
+
+/**
+ * Checkpoint for tracking transformation progress
+ */
+export interface TransformCheckpoint {
+  /** DO ID being transformed */
+  doId: string
+  /** Last successfully processed LSN */
+  lastProcessedLsn: string
+  /** Last successfully processed timestamp */
+  lastProcessedTimestamp: number
+  /** Number of entries processed since last checkpoint */
+  entriesProcessed: number
+  /** Number of Iceberg snapshots created since last checkpoint */
+  snapshotsCreated: number
+  /** Timestamp of last checkpoint */
+  checkpointedAt: number
+  /** R2 segment keys that have been fully processed */
+  processedSegmentKeys: string[]
+}
+
+/**
+ * Transformation batch result
+ */
+export interface TransformBatchResult {
+  /** Number of WAL entries processed */
+  entriesProcessed: number
+  /** Number of entries skipped (duplicates) */
+  entriesSkipped: number
+  /** Iceberg snapshot created (if any) */
+  snapshot: IcebergSnapshot | null
+  /** Processing duration in ms */
+  durationMs: number
+  /** LSN range processed */
+  lsnRange: { start: string; end: string } | null
+  /** Timestamp range processed */
+  timestampRange: { start: number; end: number } | null
+  /** R2 segment keys processed */
+  segmentKeysProcessed: string[]
+}
+
+/**
+ * Transformation statistics
+ */
+export interface TransformerStats {
+  /** Total entries processed */
+  totalEntriesProcessed: bigint
+  /** Total entries skipped (duplicates) */
+  totalEntriesSkipped: bigint
+  /** Total Iceberg snapshots created */
+  totalSnapshotsCreated: number
+  /** Total bytes written to Iceberg */
+  totalBytesWritten: bigint
+  /** Average transformation lag in ms */
+  avgTransformLagMs: number
+  /** Last transformation timestamp */
+  lastTransformAt: number | null
+  /** Last checkpoint timestamp */
+  lastCheckpointAt: number | null
+}
+
+/**
+ * Transformer configuration
+ */
+export interface TransformerConfig {
+  /** R2 bucket for WAL storage */
+  walBucket: R2Bucket
+  /** R2 bucket for Iceberg storage (can be same as walBucket) */
+  icebergBucket: R2Bucket
+  /** Durable Object ID being transformed */
+  doId: string
+  /** WAL prefix in R2 (default: 'wal/') */
+  walPrefix?: string
+  /** Iceberg table path in R2 */
+  icebergTablePath: string
+  /** Batch size for micro-batch processing (default: 1000) */
+  batchSize?: number
+  /** Maximum entries per transformation cycle (default: 10000) */
+  maxEntriesPerCycle?: number
+  /** Checkpoint interval in entries (default: 1000) */
+  checkpointInterval?: number
+  /** Target transformation lag in ms (default: 300000 = 5 minutes) */
+  targetLagMs?: number
+  /** Enable deduplication (default: true) */
+  deduplication?: boolean
+  /** DO state for storing checkpoints */
+  state?: DurableObjectState
+}
+
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG = {
+  walPrefix: 'wal/',
+  batchSize: 1000,
+  maxEntriesPerCycle: 10000,
+  checkpointInterval: 1000,
+  targetLagMs: 5 * 60 * 1000, // 5 minutes
+  deduplication: true,
+}
+
+/**
+ * Checkpoint storage key prefix
+ */
+const CHECKPOINT_KEY_PREFIX = '__wal_iceberg_checkpoint__'
+
+/**
+ * Processed LSN set storage key prefix
+ */
+const PROCESSED_LSN_KEY_PREFIX = '__wal_iceberg_processed_lsn__'
+
+/**
+ * WAL to Iceberg Transformer
+ *
+ * Transforms WAL segments from R2 into Iceberg tables with:
+ * - Continuous micro-batch processing
+ * - Exactly-once semantics via checkpointing
+ * - Deduplication of replayed entries
+ * - Schema mapping from WAL to Iceberg format
+ *
+ * @example
+ * ```typescript
+ * const transformer = new WALIcebergTransformer({
+ *   walBucket: env.WAL_BUCKET,
+ *   icebergBucket: env.ICEBERG_BUCKET,
+ *   doId: 'my-do-id',
+ *   icebergTablePath: 'iceberg/wal-table',
+ *   state: ctx.storage,
+ * })
+ *
+ * // Initialize (loads checkpoint, creates Iceberg table if needed)
+ * await transformer.initialize()
+ *
+ * // Transform pending WAL entries
+ * const result = await transformer.transformBatch()
+ *
+ * // Get stats
+ * const stats = transformer.getStats()
+ * console.log(`Processed ${stats.totalEntriesProcessed} entries`)
+ * ```
+ */
+export class WALIcebergTransformer {
+  private walBucket: R2Bucket
+  private icebergBucket: R2Bucket
+  private doId: string
+  private walPrefix: string
+  private icebergTablePath: string
+  /** @internal Reserved for future batch processing */
+  private _batchSize: number
+  private maxEntriesPerCycle: number
+  private checkpointInterval: number
+  private targetLagMs: number
+  private deduplication: boolean
+  private state: DurableObjectState | null
+
+  private icebergWriter: IcebergWALWriter | null = null
+  private checkpoint: TransformCheckpoint | null = null
+  private processedLsns: Set<string> = new Set()
+  private initialized = false
+
+  private stats: TransformerStats = {
+    totalEntriesProcessed: 0n,
+    totalEntriesSkipped: 0n,
+    totalSnapshotsCreated: 0,
+    totalBytesWritten: 0n,
+    avgTransformLagMs: 0,
+    lastTransformAt: null,
+    lastCheckpointAt: null,
+  }
+
+  private entriesSinceCheckpoint = 0
+
+  constructor(config: TransformerConfig) {
+    this.walBucket = config.walBucket
+    this.icebergBucket = config.icebergBucket
+    this.doId = config.doId
+    this.walPrefix = config.walPrefix ?? DEFAULT_CONFIG.walPrefix
+    this.icebergTablePath = config.icebergTablePath
+    this._batchSize = config.batchSize ?? DEFAULT_CONFIG.batchSize
+    this.maxEntriesPerCycle = config.maxEntriesPerCycle ?? DEFAULT_CONFIG.maxEntriesPerCycle
+    this.checkpointInterval = config.checkpointInterval ?? DEFAULT_CONFIG.checkpointInterval
+    this.targetLagMs = config.targetLagMs ?? DEFAULT_CONFIG.targetLagMs
+    this.deduplication = config.deduplication ?? DEFAULT_CONFIG.deduplication
+    this.state = config.state ?? null
+    void this._batchSize // Reserved for future batch processing
+  }
+
+  /**
+   * Initialize the transformer
+   *
+   * - Loads checkpoint from DO storage
+   * - Creates or loads Iceberg table
+   * - Initializes deduplication state
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) {
+      return
+    }
+
+    // Load checkpoint from DO storage
+    await this.loadCheckpoint()
+
+    // Initialize Iceberg writer
+    this.icebergWriter = createIcebergWALWriter({
+      bucket: this.icebergBucket,
+      tablePath: this.icebergTablePath,
+    })
+    await this.icebergWriter.initialize()
+
+    // Load processed LSNs for deduplication (keep last N LSNs in memory)
+    if (this.deduplication) {
+      await this.loadProcessedLsns()
+    }
+
+    this.initialized = true
+  }
+
+  /**
+   * Ensure the transformer is initialized
+   */
+  private ensureInitialized(): void {
+    if (!this.initialized || !this.icebergWriter) {
+      throw new Error('Transformer not initialized. Call initialize() first.')
+    }
+  }
+
+  /**
+   * Load checkpoint from DO storage
+   */
+  private async loadCheckpoint(): Promise<void> {
+    if (!this.state) {
+      this.checkpoint = this.createInitialCheckpoint()
+      return
+    }
+
+    const key = `${CHECKPOINT_KEY_PREFIX}${this.doId}`
+    const stored = await this.state.storage.get<TransformCheckpoint>(key)
+
+    if (stored) {
+      this.checkpoint = stored
+    } else {
+      this.checkpoint = this.createInitialCheckpoint()
+    }
+  }
+
+  /**
+   * Create initial checkpoint
+   */
+  private createInitialCheckpoint(): TransformCheckpoint {
+    return {
+      doId: this.doId,
+      lastProcessedLsn: '0',
+      lastProcessedTimestamp: 0,
+      entriesProcessed: 0,
+      snapshotsCreated: 0,
+      checkpointedAt: Date.now(),
+      processedSegmentKeys: [],
+    }
+  }
+
+  /**
+   * Save checkpoint to DO storage
+   */
+  private async saveCheckpoint(): Promise<void> {
+    if (!this.state || !this.checkpoint) {
+      return
+    }
+
+    const key = `${CHECKPOINT_KEY_PREFIX}${this.doId}`
+    this.checkpoint.checkpointedAt = Date.now()
+    await this.state.storage.put(key, this.checkpoint)
+
+    this.stats.lastCheckpointAt = this.checkpoint.checkpointedAt
+    this.entriesSinceCheckpoint = 0
+  }
+
+  /**
+   * Load processed LSNs for deduplication
+   */
+  private async loadProcessedLsns(): Promise<void> {
+    if (!this.state) {
+      return
+    }
+
+    const key = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`
+    const stored = await this.state.storage.get<string[]>(key)
+
+    if (stored) {
+      this.processedLsns = new Set(stored)
+    }
+  }
+
+  /**
+   * Save processed LSNs for deduplication
+   * Only keeps the most recent LSNs to prevent unbounded growth
+   */
+  private async saveProcessedLsns(): Promise<void> {
+    if (!this.state) {
+      return
+    }
+
+    // Keep only the most recent 10000 LSNs
+    const maxLsns = 10000
+    let lsns = Array.from(this.processedLsns)
+    if (lsns.length > maxLsns) {
+      // Sort by LSN value and keep the highest
+      lsns.sort((a, b) => BigInt(a) > BigInt(b) ? -1 : 1)
+      lsns = lsns.slice(0, maxLsns)
+      this.processedLsns = new Set(lsns)
+    }
+
+    const key = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`
+    await this.state.storage.put(key, lsns)
+  }
+
+  /**
+   * Check if an LSN has already been processed (for deduplication)
+   */
+  private isProcessed(lsn: string): boolean {
+    if (!this.deduplication) {
+      return false
+    }
+    return this.processedLsns.has(lsn)
+  }
+
+  /**
+   * Mark an LSN as processed
+   */
+  private markProcessed(lsn: string): void {
+    if (this.deduplication) {
+      this.processedLsns.add(lsn)
+    }
+  }
+
+  /**
+   * Get transformation statistics
+   */
+  getStats(): TransformerStats {
+    return { ...this.stats }
+  }
+
+  /**
+   * Get the current checkpoint
+   */
+  getCheckpoint(): TransformCheckpoint | null {
+    return this.checkpoint ? { ...this.checkpoint } : null
+  }
+
+  /**
+   * Get the current transformation lag in milliseconds
+   */
+  async getCurrentLag(): Promise<number> {
+    const newestSegment = await this.getNewestWALSegment()
+    if (!newestSegment) {
+      return 0
+    }
+
+    const now = Date.now()
+    return now - (this.checkpoint?.lastProcessedTimestamp ?? 0)
+  }
+
+  /**
+   * Check if transformation is within target lag
+   */
+  async isWithinTargetLag(): Promise<boolean> {
+    const lag = await this.getCurrentLag()
+    return lag <= this.targetLagMs
+  }
+
+  /**
+   * List WAL segments that need to be processed
+   */
+  private async listPendingWALSegments(): Promise<{ key: string; metadata: WALSegmentMetadata }[]> {
+    const prefix = `${this.walPrefix}${this.doId}/`
+    const processedKeys = new Set(this.checkpoint?.processedSegmentKeys ?? [])
+
+    const segments: { key: string; metadata: WALSegmentMetadata }[] = []
+    let cursor: string | undefined
+
+    do {
+      const listOptions: R2ListOptions = { prefix, limit: 1000 }
+      if (cursor) {
+        listOptions.cursor = cursor
+      }
+      const result = await this.walBucket.list(listOptions)
+
+      for (const obj of result.objects) {
+        // Skip already processed segments
+        if (processedKeys.has(obj.key)) {
+          continue
+        }
+
+        // Extract metadata from the key
+        const match = obj.key.match(/(\d+)-(\d+)\.wal$/)
+        if (match && match[1] && match[2]) {
+          const startLsn = match[1]
+          const endLsn = match[2]
+
+          // Check if this segment might contain new entries
+          // (endLsn > lastProcessedLsn)
+          if (BigInt(endLsn) > BigInt(this.checkpoint?.lastProcessedLsn ?? '0')) {
+            segments.push({
+              key: obj.key,
+              metadata: {
+                doId: this.doId,
+                startLsn,
+                endLsn,
+                entryCount: 0, // Will be populated when reading
+                uncompressedSize: 0,
+                compressedSize: obj.size,
+                compression: 'gzip',
+                checksum: '',
+                createdAt: obj.uploaded.getTime(),
+                windowStart: 0,
+                windowEnd: 0,
+              },
+            })
+          }
+        }
+      }
+
+      cursor = result.truncated ? result.cursor : undefined
+    } while (cursor)
+
+    // Sort by start LSN
+    segments.sort((a, b) => {
+      const aLsn = BigInt(a.metadata.startLsn)
+      const bLsn = BigInt(b.metadata.startLsn)
+      return aLsn < bLsn ? -1 : aLsn > bLsn ? 1 : 0
+    })
+
+    return segments
+  }
+
+  /**
+   * Get the newest WAL segment for lag calculation
+   */
+  private async getNewestWALSegment(): Promise<{ key: string; timestamp: number } | null> {
+    const prefix = `${this.walPrefix}${this.doId}/`
+
+    // List segments in reverse order (by key, which includes timestamp)
+    const result = await this.walBucket.list({ prefix, limit: 1 })
+
+    if (result.objects.length === 0) {
+      return null
+    }
+
+    const obj = result.objects[0]
+    if (obj) {
+      return {
+        key: obj.key,
+        timestamp: obj.uploaded.getTime(),
+      }
+    }
+
+    return null
+  }
+
+  /**
+   * Read a WAL segment from R2
+   */
+  private async readWALSegment(key: string): Promise<SerializedWALEntry[]> {
+    try {
+      const obj = await this.walBucket.get(key)
+      if (!obj) {
+        return []
+      }
+
+      const metadata = obj.customMetadata as Record<string, string> | undefined
+      const compression = metadata?.compression || 'none'
+
+      let data = await obj.bytes()
+
+      // Decompress if needed
+      if (compression === 'gzip') {
+        data = await this.decompressData(data)
+      }
+
+      // Parse NDJSON
+      const jsonStr = new TextDecoder().decode(data)
+      const lines = jsonStr.split('\n').filter((line) => line.trim())
+      return lines.map((line) => JSON.parse(line) as SerializedWALEntry)
+    } catch (error) {
+      transformerLogger.error('Failed to read segment', { key, error: error instanceof Error ? error.message : String(error) })
+      return []
+    }
+  }
+
+  /**
+   * Decompress gzip data
+   */
+  private async decompressData(data: Uint8Array): Promise<Uint8Array> {
+    const stream = new DecompressionStream('gzip')
+    const writer = stream.writable.getWriter()
+    const reader = stream.readable.getReader()
+
+    writer.write(data)
+    writer.close()
+
+    const chunks: Uint8Array[] = []
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      chunks.push(value)
+    }
+
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0)
+    const result = new Uint8Array(totalLength)
+    let offset = 0
+    for (const chunk of chunks) {
+      result.set(chunk, offset)
+      offset += chunk.length
+    }
+
+    return result
+  }
+
+  /**
+   * Transform WAL entries to Iceberg records
+   */
+  private transformEntries(entries: SerializedWALEntry[]): {
+    records: WALParquetRecord[]
+    skipped: number
+  } {
+    const records: WALParquetRecord[] = []
+    let skipped = 0
+
+    for (const entry of entries) {
+      // Check for duplicates
+      if (this.isProcessed(entry.lsn)) {
+        skipped++
+        continue
+      }
+
+      // Transform to Parquet record
+      const record = walEntryToParquetRecord(entry, this.doId)
+      records.push(record)
+
+      // Mark as processed
+      this.markProcessed(entry.lsn)
+    }
+
+    return { records, skipped }
+  }
+
+  /**
+   * Transform a batch of pending WAL entries to Iceberg
+   *
+   * This is the main transformation method. Call it periodically
+   * (e.g., via a scheduled worker or alarm) to keep transformation
+   * lag within target.
+   */
+  async transformBatch(): Promise<TransformBatchResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+    let entriesProcessed = 0
+    let entriesSkipped = 0
+    let lsnRange: { start: string; end: string } | null = null
+    let timestampRange: { start: number; end: number } | null = null
+    const segmentKeysProcessed: string[] = []
+
+    // Get pending WAL segments
+    const pendingSegments = await this.listPendingWALSegments()
+
+    if (pendingSegments.length === 0) {
+      return {
+        entriesProcessed: 0,
+        entriesSkipped: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+        lsnRange: null,
+        timestampRange: null,
+        segmentKeysProcessed: [],
+      }
+    }
+
+    // Process segments in order, respecting maxEntriesPerCycle
+    for (const segment of pendingSegments) {
+      if (entriesProcessed >= this.maxEntriesPerCycle) {
+        break
+      }
+
+      // Read segment entries
+      const entries = await this.readWALSegment(segment.key)
+      if (entries.length === 0) {
+        continue
+      }
+
+      // Filter to entries after lastProcessedLsn
+      let newEntries = entries.filter(
+        (e) => BigInt(e.lsn) > BigInt(this.checkpoint?.lastProcessedLsn ?? '0')
+      )
+
+      if (newEntries.length === 0) {
+        // Segment fully processed, mark it
+        segmentKeysProcessed.push(segment.key)
+        continue
+      }
+
+      // Limit entries to respect maxEntriesPerCycle
+      const remainingCapacity = this.maxEntriesPerCycle - entriesProcessed
+      const segmentFullyProcessed = newEntries.length <= remainingCapacity
+      if (newEntries.length > remainingCapacity) {
+        newEntries = newEntries.slice(0, remainingCapacity)
+      }
+
+      // Transform entries
+      const { records, skipped } = this.transformEntries(newEntries)
+      entriesSkipped += skipped
+
+      if (records.length > 0) {
+        // Add to Iceberg writer
+        this.icebergWriter!.addEntries(
+          newEntries.filter((e) => !this.isProcessed(e.lsn) || records.some((r) => r.lsn === e.lsn)),
+          this.doId
+        )
+
+        entriesProcessed += records.length
+
+        // Update LSN and timestamp ranges
+        const firstEntry = newEntries[0]
+        const lastEntry = newEntries[newEntries.length - 1]
+
+        if (firstEntry && (!lsnRange || BigInt(firstEntry.lsn) < BigInt(lsnRange.start))) {
+          lsnRange = lsnRange ?? { start: firstEntry.lsn, end: lastEntry!.lsn }
+          lsnRange.start = firstEntry.lsn
+        }
+        if (lastEntry && (!lsnRange || BigInt(lastEntry.lsn) > BigInt(lsnRange.end))) {
+          lsnRange = lsnRange ?? { start: firstEntry!.lsn, end: lastEntry.lsn }
+          lsnRange.end = lastEntry.lsn
+        }
+
+        if (firstEntry && (!timestampRange || firstEntry.timestamp < timestampRange.start)) {
+          timestampRange = timestampRange ?? { start: firstEntry.timestamp, end: lastEntry!.timestamp }
+          timestampRange.start = firstEntry.timestamp
+        }
+        if (lastEntry && (!timestampRange || lastEntry.timestamp > timestampRange.end)) {
+          timestampRange = timestampRange ?? { start: firstEntry!.timestamp, end: lastEntry.timestamp }
+          timestampRange.end = lastEntry.timestamp
+        }
+
+        // Update checkpoint progress
+        if (lastEntry) {
+          this.checkpoint!.lastProcessedLsn = lastEntry.lsn
+          this.checkpoint!.lastProcessedTimestamp = lastEntry.timestamp
+        }
+      }
+
+      // Only mark segment as fully processed if we processed all its entries
+      if (segmentFullyProcessed) {
+        segmentKeysProcessed.push(segment.key)
+      }
+      this.entriesSinceCheckpoint += records.length
+
+      // Checkpoint if needed
+      if (this.entriesSinceCheckpoint >= this.checkpointInterval) {
+        await this.saveCheckpoint()
+        await this.saveProcessedLsns()
+      }
+    }
+
+    // Flush to Iceberg and create snapshot
+    let snapshot: IcebergSnapshot | null = null
+    if (entriesProcessed > 0) {
+      snapshot = await this.icebergWriter!.flush()
+
+      if (snapshot) {
+        this.stats.totalSnapshotsCreated++
+        this.checkpoint!.snapshotsCreated++
+      }
+    }
+
+    // Update checkpoint with processed segments
+    this.checkpoint!.processedSegmentKeys.push(...segmentKeysProcessed)
+    this.checkpoint!.entriesProcessed += entriesProcessed
+
+    // Final checkpoint save
+    await this.saveCheckpoint()
+    await this.saveProcessedLsns()
+
+    // Update stats
+    this.stats.totalEntriesProcessed += BigInt(entriesProcessed)
+    this.stats.totalEntriesSkipped += BigInt(entriesSkipped)
+    this.stats.lastTransformAt = Date.now()
+
+    if (this.icebergWriter) {
+      const writerStats = this.icebergWriter.getStats()
+      this.stats.totalBytesWritten = writerStats.bytesWritten
+    }
+
+    // Calculate average lag
+    if (timestampRange) {
+      const lag = Date.now() - timestampRange.end
+      this.stats.avgTransformLagMs = (this.stats.avgTransformLagMs + lag) / 2
+    }
+
+    const durationMs = performance.now() - startTime
+
+    return {
+      entriesProcessed,
+      entriesSkipped,
+      snapshot,
+      durationMs,
+      lsnRange,
+      timestampRange,
+      segmentKeysProcessed,
+    }
+  }
+
+  /**
+   * Transform all pending WAL entries until caught up
+   *
+   * Continues transforming batches until there are no more
+   * pending entries or the target lag is achieved.
+   */
+  async transformUntilCaughtUp(
+    maxIterations = 100
+  ): Promise<{
+    totalEntriesProcessed: number
+    totalEntriesSkipped: number
+    totalSnapshots: number
+    iterations: number
+    durationMs: number
+  }> {
+    const startTime = performance.now()
+    let totalEntriesProcessed = 0
+    let totalEntriesSkipped = 0
+    let totalSnapshots = 0
+    let iterations = 0
+
+    for (let i = 0; i < maxIterations; i++) {
+      const result = await this.transformBatch()
+      iterations++
+
+      totalEntriesProcessed += result.entriesProcessed
+      totalEntriesSkipped += result.entriesSkipped
+      if (result.snapshot) {
+        totalSnapshots++
+      }
+
+      // Stop if no entries were processed (caught up)
+      if (result.entriesProcessed === 0) {
+        break
+      }
+
+      // Stop if within target lag
+      if (await this.isWithinTargetLag()) {
+        break
+      }
+    }
+
+    return {
+      totalEntriesProcessed,
+      totalEntriesSkipped,
+      totalSnapshots,
+      iterations,
+      durationMs: performance.now() - startTime,
+    }
+  }
+
+  /**
+   * Reset the transformer checkpoint
+   *
+   * WARNING: This will cause all WAL entries to be reprocessed.
+   * Use only for disaster recovery or testing.
+   */
+  async resetCheckpoint(): Promise<void> {
+    this.checkpoint = this.createInitialCheckpoint()
+    this.processedLsns.clear()
+    this.entriesSinceCheckpoint = 0
+
+    if (this.state) {
+      const checkpointKey = `${CHECKPOINT_KEY_PREFIX}${this.doId}`
+      const processedKey = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`
+      await this.state.storage.delete([checkpointKey, processedKey])
+    }
+
+    this.stats = {
+      totalEntriesProcessed: 0n,
+      totalEntriesSkipped: 0n,
+      totalSnapshotsCreated: 0,
+      totalBytesWritten: 0n,
+      avgTransformLagMs: 0,
+      lastTransformAt: null,
+      lastCheckpointAt: null,
+    }
+  }
+
+  /**
+   * Get the Iceberg table metadata
+   */
+  getIcebergMetadata(): IcebergTableMetadata | null {
+    return this.icebergWriter?.getMetadata() ?? null
+  }
+
+  /**
+   * Get the Iceberg writer statistics
+   */
+  getIcebergWriterStats() {
+    return this.icebergWriter?.getStats() ?? null
+  }
+}
+
+/**
+ * Create a WAL to Iceberg transformer
+ *
+ * @example
+ * ```typescript
+ * const transformer = createWALIcebergTransformer({
+ *   walBucket: env.WAL_BUCKET,
+ *   icebergBucket: env.ICEBERG_BUCKET,
+ *   doId: 'my-do-id',
+ *   icebergTablePath: 'iceberg/wal-table',
+ *   state: ctx.storage,
+ * })
+ *
+ * await transformer.initialize()
+ * const result = await transformer.transformBatch()
+ * ```
+ */
+export function createWALIcebergTransformer(config: TransformerConfig): WALIcebergTransformer {
+  return new WALIcebergTransformer(config)
+}
+
+/**
+ * Schema mapper for WAL entries to custom Iceberg schemas
+ *
+ * Allows mapping WAL entries to user-defined Iceberg schemas
+ * for per-table data warehousing.
+ */
+export interface WALSchemaMapper {
+  /** Map a WAL entry to a custom record format */
+  mapEntry(entry: SerializedWALEntry, doId: string): Record<string, unknown>
+  /** Get the Iceberg schema for the mapped records */
+  getSchema(): IcebergSchema
+  /** Get the partition spec for the mapped records */
+  getPartitionSpec(): IcebergPartitionSpec
+}
+
+/**
+ * Default WAL schema mapper (uses the standard WAL schema)
+ */
+export class DefaultWALSchemaMapper implements WALSchemaMapper {
+  mapEntry(entry: SerializedWALEntry, doId: string): Record<string, unknown> {
+    return walEntryToParquetRecord(entry, doId) as unknown as Record<string, unknown>
+  }
+
+  getSchema(): IcebergSchema {
+    return createWALSchema()
+  }
+
+  getPartitionSpec(): IcebergPartitionSpec {
+    return createWALPartitionSpec()
+  }
+}
+
+/**
+ * Per-table schema mapper
+ *
+ * Maps WAL entries to table-specific Iceberg tables,
+ * extracting row data into proper columns.
+ */
+export class TableSchemaMapper implements WALSchemaMapper {
+  private tableName: string
+  private schema: IcebergSchema
+  private partitionSpec: IcebergPartitionSpec
+
+  constructor(
+    tableName: string,
+    schema: IcebergSchema,
+    partitionSpec: IcebergPartitionSpec
+  ) {
+    this.tableName = tableName
+    this.schema = schema
+    this.partitionSpec = partitionSpec
+  }
+
+  mapEntry(entry: SerializedWALEntry, doId: string): Record<string, unknown> {
+    // Only map entries for this table
+    if (entry.table !== this.tableName) {
+      throw new Error(`Entry is for table ${entry.table}, expected ${this.tableName}`)
+    }
+
+    // Extract row data
+    const rowData = entry.operation === 'DELETE' ? entry.oldRow : entry.newRow
+    if (!rowData) {
+      return {
+        __do_id: doId,
+        __lsn: entry.lsn,
+        __timestamp: entry.timestamp,
+        __operation: entry.operation,
+      }
+    }
+
+    // Map row columns to schema fields
+    const mapped: Record<string, unknown> = {
+      __do_id: doId,
+      __lsn: entry.lsn,
+      __timestamp: entry.timestamp,
+      __operation: entry.operation,
+    }
+
+    for (const field of this.schema.fields) {
+      const value = rowData[field.name]
+      if (value !== undefined) {
+        mapped[field.name] = value
+      }
+    }
+
+    return mapped
+  }
+
+  getSchema(): IcebergSchema {
+    return this.schema
+  }
+
+  getPartitionSpec(): IcebergPartitionSpec {
+    return this.partitionSpec
+  }
+}
+
+/**
+ * Multi-table transformer that routes WAL entries to table-specific Iceberg tables
+ */
+export class MultiTableTransformer {
+  private transformers: Map<string, WALIcebergTransformer> = new Map()
+  private defaultTransformer: WALIcebergTransformer
+  private walBucket: R2Bucket
+  private icebergBucket: R2Bucket
+  private doId: string
+  private state: DurableObjectState | null
+
+  constructor(config: {
+    walBucket: R2Bucket
+    icebergBucket: R2Bucket
+    doId: string
+    defaultTablePath: string
+    state?: DurableObjectState
+  }) {
+    this.walBucket = config.walBucket
+    this.icebergBucket = config.icebergBucket
+    this.doId = config.doId
+    this.state = config.state ?? null
+
+    // Create default transformer for unmapped tables
+    const defaultConfig: TransformerConfig = {
+      walBucket: config.walBucket,
+      icebergBucket: config.icebergBucket,
+      doId: config.doId,
+      icebergTablePath: config.defaultTablePath,
+    }
+    if (config.state !== undefined) {
+      defaultConfig.state = config.state
+    }
+    this.defaultTransformer = createWALIcebergTransformer(defaultConfig)
+  }
+
+  /**
+   * Register a table-specific transformer
+   */
+  registerTable(tableName: string, icebergTablePath: string): void {
+    const transformerConfig: TransformerConfig = {
+      walBucket: this.walBucket,
+      icebergBucket: this.icebergBucket,
+      doId: this.doId,
+      icebergTablePath,
+    }
+    if (this.state !== null) {
+      transformerConfig.state = this.state
+    }
+    const transformer = createWALIcebergTransformer(transformerConfig)
+    this.transformers.set(tableName, transformer)
+  }
+
+  /**
+   * Initialize all transformers
+   */
+  async initialize(): Promise<void> {
+    await this.defaultTransformer.initialize()
+    for (const transformer of this.transformers.values()) {
+      await transformer.initialize()
+    }
+  }
+
+  /**
+   * Get transformer for a specific table
+   */
+  getTransformer(tableName: string): WALIcebergTransformer {
+    return this.transformers.get(tableName) ?? this.defaultTransformer
+  }
+}