@dotdo/postgres 0.1.0

package/dist/iceberg/transformer.js

@@ -0,0 +1,770 @@
+/**
+ * WAL to Iceberg Transformer
+ * Task: postgres-5iv - WAL to Iceberg transformation
+ *
+ * Transforms streaming WAL data into Iceberg table format for efficient querying.
+ * Implements:
+ * - Continuous transformation pipeline
+ * - Batched micro-batch processing
+ * - Schema mapping from WAL to Iceberg
+ * - Deduplication of replayed/retried WAL
+ * - Exactly-once processing semantics via checkpointing
+ */
+import { createLogger, LogLevel } from '@dotdo/postgres-shared';
+import { createIcebergWALWriter } from './writer';
+import { walEntryToParquetRecord } from './parquet';
+import { createWALSchema, createWALPartitionSpec } from './schema';
+// Re-export schema utils for external use
+export { getPartitionValue, generatePartitionPath } from './schema';
+// Module-level logger for transformer errors
+const transformerLogger = createLogger({
+    level: LogLevel.ERROR,
+    prefix: '[WALIcebergTransformer]',
+});
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG = {
+    walPrefix: 'wal/',
+    batchSize: 1000,
+    maxEntriesPerCycle: 10000,
+    checkpointInterval: 1000,
+    targetLagMs: 5 * 60 * 1000, // 5 minutes
+    deduplication: true,
+};
+/**
+ * Checkpoint storage key prefix
+ */
+const CHECKPOINT_KEY_PREFIX = '__wal_iceberg_checkpoint__';
+/**
+ * Processed LSN set storage key prefix
+ */
+const PROCESSED_LSN_KEY_PREFIX = '__wal_iceberg_processed_lsn__';
+/**
+ * WAL to Iceberg Transformer
+ *
+ * Transforms WAL segments from R2 into Iceberg tables with:
+ * - Continuous micro-batch processing
+ * - Exactly-once semantics via checkpointing
+ * - Deduplication of replayed entries
+ * - Schema mapping from WAL to Iceberg format
+ *
+ * @example
+ * ```typescript
+ * const transformer = new WALIcebergTransformer({
+ *   walBucket: env.WAL_BUCKET,
+ *   icebergBucket: env.ICEBERG_BUCKET,
+ *   doId: 'my-do-id',
+ *   icebergTablePath: 'iceberg/wal-table',
+ *   state: ctx.storage,
+ * })
+ *
+ * // Initialize (loads checkpoint, creates Iceberg table if needed)
+ * await transformer.initialize()
+ *
+ * // Transform pending WAL entries
+ * const result = await transformer.transformBatch()
+ *
+ * // Get stats
+ * const stats = transformer.getStats()
+ * console.log(`Processed ${stats.totalEntriesProcessed} entries`)
+ * ```
+ */
+export class WALIcebergTransformer {
+    walBucket;
+    icebergBucket;
+    doId;
+    walPrefix;
+    icebergTablePath;
+    /** @internal Reserved for future batch processing */
+    _batchSize;
+    maxEntriesPerCycle;
+    checkpointInterval;
+    targetLagMs;
+    deduplication;
+    state;
+    icebergWriter = null;
+    checkpoint = null;
+    processedLsns = new Set();
+    initialized = false;
+    stats = {
+        totalEntriesProcessed: 0n,
+        totalEntriesSkipped: 0n,
+        totalSnapshotsCreated: 0,
+        totalBytesWritten: 0n,
+        avgTransformLagMs: 0,
+        lastTransformAt: null,
+        lastCheckpointAt: null,
+    };
+    entriesSinceCheckpoint = 0;
+    constructor(config) {
+        this.walBucket = config.walBucket;
+        this.icebergBucket = config.icebergBucket;
+        this.doId = config.doId;
+        this.walPrefix = config.walPrefix ?? DEFAULT_CONFIG.walPrefix;
+        this.icebergTablePath = config.icebergTablePath;
+        this._batchSize = config.batchSize ?? DEFAULT_CONFIG.batchSize;
+        this.maxEntriesPerCycle = config.maxEntriesPerCycle ?? DEFAULT_CONFIG.maxEntriesPerCycle;
+        this.checkpointInterval = config.checkpointInterval ?? DEFAULT_CONFIG.checkpointInterval;
+        this.targetLagMs = config.targetLagMs ?? DEFAULT_CONFIG.targetLagMs;
+        this.deduplication = config.deduplication ?? DEFAULT_CONFIG.deduplication;
+        this.state = config.state ?? null;
+        void this._batchSize; // Reserved for future batch processing
+    }
+    /**
+     * Initialize the transformer
+     *
+     * - Loads checkpoint from DO storage
+     * - Creates or loads Iceberg table
+     * - Initializes deduplication state
+     */
+    async initialize() {
+        if (this.initialized) {
+            return;
+        }
+        // Load checkpoint from DO storage
+        await this.loadCheckpoint();
+        // Initialize Iceberg writer
+        this.icebergWriter = createIcebergWALWriter({
+            bucket: this.icebergBucket,
+            tablePath: this.icebergTablePath,
+        });
+        await this.icebergWriter.initialize();
+        // Load processed LSNs for deduplication (keep last N LSNs in memory)
+        if (this.deduplication) {
+            await this.loadProcessedLsns();
+        }
+        this.initialized = true;
+    }
+    /**
+     * Ensure the transformer is initialized
+     */
+    ensureInitialized() {
+        if (!this.initialized || !this.icebergWriter) {
+            throw new Error('Transformer not initialized. Call initialize() first.');
+        }
+    }
+    /**
+     * Load checkpoint from DO storage
+     */
+    async loadCheckpoint() {
+        if (!this.state) {
+            this.checkpoint = this.createInitialCheckpoint();
+            return;
+        }
+        const key = `${CHECKPOINT_KEY_PREFIX}${this.doId}`;
+        const stored = await this.state.storage.get(key);
+        if (stored) {
+            this.checkpoint = stored;
+        }
+        else {
+            this.checkpoint = this.createInitialCheckpoint();
+        }
+    }
+    /**
+     * Create initial checkpoint
+     */
+    createInitialCheckpoint() {
+        return {
+            doId: this.doId,
+            lastProcessedLsn: '0',
+            lastProcessedTimestamp: 0,
+            entriesProcessed: 0,
+            snapshotsCreated: 0,
+            checkpointedAt: Date.now(),
+            processedSegmentKeys: [],
+        };
+    }
+    /**
+     * Save checkpoint to DO storage
+     */
+    async saveCheckpoint() {
+        if (!this.state || !this.checkpoint) {
+            return;
+        }
+        const key = `${CHECKPOINT_KEY_PREFIX}${this.doId}`;
+        this.checkpoint.checkpointedAt = Date.now();
+        await this.state.storage.put(key, this.checkpoint);
+        this.stats.lastCheckpointAt = this.checkpoint.checkpointedAt;
+        this.entriesSinceCheckpoint = 0;
+    }
+    /**
+     * Load processed LSNs for deduplication
+     */
+    async loadProcessedLsns() {
+        if (!this.state) {
+            return;
+        }
+        const key = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`;
+        const stored = await this.state.storage.get(key);
+        if (stored) {
+            this.processedLsns = new Set(stored);
+        }
+    }
+    /**
+     * Save processed LSNs for deduplication
+     * Only keeps the most recent LSNs to prevent unbounded growth
+     */
+    async saveProcessedLsns() {
+        if (!this.state) {
+            return;
+        }
+        // Keep only the most recent 10000 LSNs
+        const maxLsns = 10000;
+        let lsns = Array.from(this.processedLsns);
+        if (lsns.length > maxLsns) {
+            // Sort by LSN value and keep the highest
+            lsns.sort((a, b) => BigInt(a) > BigInt(b) ? -1 : 1);
+            lsns = lsns.slice(0, maxLsns);
+            this.processedLsns = new Set(lsns);
+        }
+        const key = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`;
+        await this.state.storage.put(key, lsns);
+    }
+    /**
+     * Check if an LSN has already been processed (for deduplication)
+     */
+    isProcessed(lsn) {
+        if (!this.deduplication) {
+            return false;
+        }
+        return this.processedLsns.has(lsn);
+    }
+    /**
+     * Mark an LSN as processed
+     */
+    markProcessed(lsn) {
+        if (this.deduplication) {
+            this.processedLsns.add(lsn);
+        }
+    }
+    /**
+     * Get transformation statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Get the current checkpoint
+     */
+    getCheckpoint() {
+        return this.checkpoint ? { ...this.checkpoint } : null;
+    }
+    /**
+     * Get the current transformation lag in milliseconds
+     */
+    async getCurrentLag() {
+        const newestSegment = await this.getNewestWALSegment();
+        if (!newestSegment) {
+            return 0;
+        }
+        const now = Date.now();
+        return now - (this.checkpoint?.lastProcessedTimestamp ?? 0);
+    }
+    /**
+     * Check if transformation is within target lag
+     */
+    async isWithinTargetLag() {
+        const lag = await this.getCurrentLag();
+        return lag <= this.targetLagMs;
+    }
+    /**
+     * List WAL segments that need to be processed
+     */
+    async listPendingWALSegments() {
+        const prefix = `${this.walPrefix}${this.doId}/`;
+        const processedKeys = new Set(this.checkpoint?.processedSegmentKeys ?? []);
+        const segments = [];
+        let cursor;
+        do {
+            const listOptions = { prefix, limit: 1000 };
+            if (cursor) {
+                listOptions.cursor = cursor;
+            }
+            const result = await this.walBucket.list(listOptions);
+            for (const obj of result.objects) {
+                // Skip already processed segments
+                if (processedKeys.has(obj.key)) {
+                    continue;
+                }
+                // Extract metadata from the key
+                const match = obj.key.match(/(\d+)-(\d+)\.wal$/);
+                if (match && match[1] && match[2]) {
+                    const startLsn = match[1];
+                    const endLsn = match[2];
+                    // Check if this segment might contain new entries
+                    // (endLsn > lastProcessedLsn)
+                    if (BigInt(endLsn) > BigInt(this.checkpoint?.lastProcessedLsn ?? '0')) {
+                        segments.push({
+                            key: obj.key,
+                            metadata: {
+                                doId: this.doId,
+                                startLsn,
+                                endLsn,
+                                entryCount: 0, // Will be populated when reading
+                                uncompressedSize: 0,
+                                compressedSize: obj.size,
+                                compression: 'gzip',
+                                checksum: '',
+                                createdAt: obj.uploaded.getTime(),
+                                windowStart: 0,
+                                windowEnd: 0,
+                            },
+                        });
+                    }
+                }
+            }
+            cursor = result.truncated ? result.cursor : undefined;
+        } while (cursor);
+        // Sort by start LSN
+        segments.sort((a, b) => {
+            const aLsn = BigInt(a.metadata.startLsn);
+            const bLsn = BigInt(b.metadata.startLsn);
+            return aLsn < bLsn ? -1 : aLsn > bLsn ? 1 : 0;
+        });
+        return segments;
+    }
+    /**
+     * Get the newest WAL segment for lag calculation
+     */
+    async getNewestWALSegment() {
+        const prefix = `${this.walPrefix}${this.doId}/`;
+        // List segments in reverse order (by key, which includes timestamp)
+        const result = await this.walBucket.list({ prefix, limit: 1 });
+        if (result.objects.length === 0) {
+            return null;
+        }
+        const obj = result.objects[0];
+        if (obj) {
+            return {
+                key: obj.key,
+                timestamp: obj.uploaded.getTime(),
+            };
+        }
+        return null;
+    }
+    /**
+     * Read a WAL segment from R2
+     */
+    async readWALSegment(key) {
+        try {
+            const obj = await this.walBucket.get(key);
+            if (!obj) {
+                return [];
+            }
+            const metadata = obj.customMetadata;
+            const compression = metadata?.compression || 'none';
+            let data = await obj.bytes();
+            // Decompress if needed
+            if (compression === 'gzip') {
+                data = await this.decompressData(data);
+            }
+            // Parse NDJSON
+            const jsonStr = new TextDecoder().decode(data);
+            const lines = jsonStr.split('\n').filter((line) => line.trim());
+            return lines.map((line) => JSON.parse(line));
+        }
+        catch (error) {
+            transformerLogger.error('Failed to read segment', { key, error: error instanceof Error ? error.message : String(error) });
+            return [];
+        }
+    }
+    /**
+     * Decompress gzip data
+     */
+    async decompressData(data) {
+        const stream = new DecompressionStream('gzip');
+        const writer = stream.writable.getWriter();
+        const reader = stream.readable.getReader();
+        writer.write(data);
+        writer.close();
+        const chunks = [];
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            chunks.push(value);
+        }
+        const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+        const result = new Uint8Array(totalLength);
+        let offset = 0;
+        for (const chunk of chunks) {
+            result.set(chunk, offset);
+            offset += chunk.length;
+        }
+        return result;
+    }
+    /**
+     * Transform WAL entries to Iceberg records
+     */
+    transformEntries(entries) {
+        const records = [];
+        let skipped = 0;
+        for (const entry of entries) {
+            // Check for duplicates
+            if (this.isProcessed(entry.lsn)) {
+                skipped++;
+                continue;
+            }
+            // Transform to Parquet record
+            const record = walEntryToParquetRecord(entry, this.doId);
+            records.push(record);
+            // Mark as processed
+            this.markProcessed(entry.lsn);
+        }
+        return { records, skipped };
+    }
+    /**
+     * Transform a batch of pending WAL entries to Iceberg
+     *
+     * This is the main transformation method. Call it periodically
+     * (e.g., via a scheduled worker or alarm) to keep transformation
+     * lag within target.
+     */
+    async transformBatch() {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        let entriesProcessed = 0;
+        let entriesSkipped = 0;
+        let lsnRange = null;
+        let timestampRange = null;
+        const segmentKeysProcessed = [];
+        // Get pending WAL segments
+        const pendingSegments = await this.listPendingWALSegments();
+        if (pendingSegments.length === 0) {
+            return {
+                entriesProcessed: 0,
+                entriesSkipped: 0,
+                snapshot: null,
+                durationMs: performance.now() - startTime,
+                lsnRange: null,
+                timestampRange: null,
+                segmentKeysProcessed: [],
+            };
+        }
+        // Process segments in order, respecting maxEntriesPerCycle
+        for (const segment of pendingSegments) {
+            if (entriesProcessed >= this.maxEntriesPerCycle) {
+                break;
+            }
+            // Read segment entries
+            const entries = await this.readWALSegment(segment.key);
+            if (entries.length === 0) {
+                continue;
+            }
+            // Filter to entries after lastProcessedLsn
+            let newEntries = entries.filter((e) => BigInt(e.lsn) > BigInt(this.checkpoint?.lastProcessedLsn ?? '0'));
+            if (newEntries.length === 0) {
+                // Segment fully processed, mark it
+                segmentKeysProcessed.push(segment.key);
+                continue;
+            }
+            // Limit entries to respect maxEntriesPerCycle
+            const remainingCapacity = this.maxEntriesPerCycle - entriesProcessed;
+            const segmentFullyProcessed = newEntries.length <= remainingCapacity;
+            if (newEntries.length > remainingCapacity) {
+                newEntries = newEntries.slice(0, remainingCapacity);
+            }
+            // Transform entries
+            const { records, skipped } = this.transformEntries(newEntries);
+            entriesSkipped += skipped;
+            if (records.length > 0) {
+                // Add to Iceberg writer
+                this.icebergWriter.addEntries(newEntries.filter((e) => !this.isProcessed(e.lsn) || records.some((r) => r.lsn === e.lsn)), this.doId);
+                entriesProcessed += records.length;
+                // Update LSN and timestamp ranges
+                const firstEntry = newEntries[0];
+                const lastEntry = newEntries[newEntries.length - 1];
+                if (firstEntry && (!lsnRange || BigInt(firstEntry.lsn) < BigInt(lsnRange.start))) {
+                    lsnRange = lsnRange ?? { start: firstEntry.lsn, end: lastEntry.lsn };
+                    lsnRange.start = firstEntry.lsn;
+                }
+                if (lastEntry && (!lsnRange || BigInt(lastEntry.lsn) > BigInt(lsnRange.end))) {
+                    lsnRange = lsnRange ?? { start: firstEntry.lsn, end: lastEntry.lsn };
+                    lsnRange.end = lastEntry.lsn;
+                }
+                if (firstEntry && (!timestampRange || firstEntry.timestamp < timestampRange.start)) {
+                    timestampRange = timestampRange ?? { start: firstEntry.timestamp, end: lastEntry.timestamp };
+                    timestampRange.start = firstEntry.timestamp;
+                }
+                if (lastEntry && (!timestampRange || lastEntry.timestamp > timestampRange.end)) {
+                    timestampRange = timestampRange ?? { start: firstEntry.timestamp, end: lastEntry.timestamp };
+                    timestampRange.end = lastEntry.timestamp;
+                }
+                // Update checkpoint progress
+                if (lastEntry) {
+                    this.checkpoint.lastProcessedLsn = lastEntry.lsn;
+                    this.checkpoint.lastProcessedTimestamp = lastEntry.timestamp;
+                }
+            }
+            // Only mark segment as fully processed if we processed all its entries
+            if (segmentFullyProcessed) {
+                segmentKeysProcessed.push(segment.key);
+            }
+            this.entriesSinceCheckpoint += records.length;
+            // Checkpoint if needed
+            if (this.entriesSinceCheckpoint >= this.checkpointInterval) {
+                await this.saveCheckpoint();
+                await this.saveProcessedLsns();
+            }
+        }
+        // Flush to Iceberg and create snapshot
+        let snapshot = null;
+        if (entriesProcessed > 0) {
+            snapshot = await this.icebergWriter.flush();
+            if (snapshot) {
+                this.stats.totalSnapshotsCreated++;
+                this.checkpoint.snapshotsCreated++;
+            }
+        }
+        // Update checkpoint with processed segments
+        this.checkpoint.processedSegmentKeys.push(...segmentKeysProcessed);
+        this.checkpoint.entriesProcessed += entriesProcessed;
+        // Final checkpoint save
+        await this.saveCheckpoint();
+        await this.saveProcessedLsns();
+        // Update stats
+        this.stats.totalEntriesProcessed += BigInt(entriesProcessed);
+        this.stats.totalEntriesSkipped += BigInt(entriesSkipped);
+        this.stats.lastTransformAt = Date.now();
+        if (this.icebergWriter) {
+            const writerStats = this.icebergWriter.getStats();
+            this.stats.totalBytesWritten = writerStats.bytesWritten;
+        }
+        // Calculate average lag
+        if (timestampRange) {
+            const lag = Date.now() - timestampRange.end;
+            this.stats.avgTransformLagMs = (this.stats.avgTransformLagMs + lag) / 2;
+        }
+        const durationMs = performance.now() - startTime;
+        return {
+            entriesProcessed,
+            entriesSkipped,
+            snapshot,
+            durationMs,
+            lsnRange,
+            timestampRange,
+            segmentKeysProcessed,
+        };
+    }
+    /**
+     * Transform all pending WAL entries until caught up
+     *
+     * Continues transforming batches until there are no more
+     * pending entries or the target lag is achieved.
+     */
+    async transformUntilCaughtUp(maxIterations = 100) {
+        const startTime = performance.now();
+        let totalEntriesProcessed = 0;
+        let totalEntriesSkipped = 0;
+        let totalSnapshots = 0;
+        let iterations = 0;
+        for (let i = 0; i < maxIterations; i++) {
+            const result = await this.transformBatch();
+            iterations++;
+            totalEntriesProcessed += result.entriesProcessed;
+            totalEntriesSkipped += result.entriesSkipped;
+            if (result.snapshot) {
+                totalSnapshots++;
+            }
+            // Stop if no entries were processed (caught up)
+            if (result.entriesProcessed === 0) {
+                break;
+            }
+            // Stop if within target lag
+            if (await this.isWithinTargetLag()) {
+                break;
+            }
+        }
+        return {
+            totalEntriesProcessed,
+            totalEntriesSkipped,
+            totalSnapshots,
+            iterations,
+            durationMs: performance.now() - startTime,
+        };
+    }
+    /**
+     * Reset the transformer checkpoint
+     *
+     * WARNING: This will cause all WAL entries to be reprocessed.
+     * Use only for disaster recovery or testing.
+     */
+    async resetCheckpoint() {
+        this.checkpoint = this.createInitialCheckpoint();
+        this.processedLsns.clear();
+        this.entriesSinceCheckpoint = 0;
+        if (this.state) {
+            const checkpointKey = `${CHECKPOINT_KEY_PREFIX}${this.doId}`;
+            const processedKey = `${PROCESSED_LSN_KEY_PREFIX}${this.doId}`;
+            await this.state.storage.delete([checkpointKey, processedKey]);
+        }
+        this.stats = {
+            totalEntriesProcessed: 0n,
+            totalEntriesSkipped: 0n,
+            totalSnapshotsCreated: 0,
+            totalBytesWritten: 0n,
+            avgTransformLagMs: 0,
+            lastTransformAt: null,
+            lastCheckpointAt: null,
+        };
+    }
+    /**
+     * Get the Iceberg table metadata
+     */
+    getIcebergMetadata() {
+        return this.icebergWriter?.getMetadata() ?? null;
+    }
+    /**
+     * Get the Iceberg writer statistics
+     */
+    getIcebergWriterStats() {
+        return this.icebergWriter?.getStats() ?? null;
+    }
+}
+/**
+ * Create a WAL to Iceberg transformer
+ *
+ * @example
+ * ```typescript
+ * const transformer = createWALIcebergTransformer({
+ *   walBucket: env.WAL_BUCKET,
+ *   icebergBucket: env.ICEBERG_BUCKET,
+ *   doId: 'my-do-id',
+ *   icebergTablePath: 'iceberg/wal-table',
+ *   state: ctx.storage,
+ * })
+ *
+ * await transformer.initialize()
+ * const result = await transformer.transformBatch()
+ * ```
+ */
+export function createWALIcebergTransformer(config) {
+    return new WALIcebergTransformer(config);
+}
+/**
+ * Default WAL schema mapper (uses the standard WAL schema)
+ */
+export class DefaultWALSchemaMapper {
+    mapEntry(entry, doId) {
+        return walEntryToParquetRecord(entry, doId);
+    }
+    getSchema() {
+        return createWALSchema();
+    }
+    getPartitionSpec() {
+        return createWALPartitionSpec();
+    }
+}
+/**
+ * Per-table schema mapper
+ *
+ * Maps WAL entries to table-specific Iceberg tables,
+ * extracting row data into proper columns.
+ */
+export class TableSchemaMapper {
+    tableName;
+    schema;
+    partitionSpec;
+    constructor(tableName, schema, partitionSpec) {
+        this.tableName = tableName;
+        this.schema = schema;
+        this.partitionSpec = partitionSpec;
+    }
+    mapEntry(entry, doId) {
+        // Only map entries for this table
+        if (entry.table !== this.tableName) {
+            throw new Error(`Entry is for table ${entry.table}, expected ${this.tableName}`);
+        }
+        // Extract row data
+        const rowData = entry.operation === 'DELETE' ? entry.oldRow : entry.newRow;
+        if (!rowData) {
+            return {
+                __do_id: doId,
+                __lsn: entry.lsn,
+                __timestamp: entry.timestamp,
+                __operation: entry.operation,
+            };
+        }
+        // Map row columns to schema fields
+        const mapped = {
+            __do_id: doId,
+            __lsn: entry.lsn,
+            __timestamp: entry.timestamp,
+            __operation: entry.operation,
+        };
+        for (const field of this.schema.fields) {
+            const value = rowData[field.name];
+            if (value !== undefined) {
+                mapped[field.name] = value;
+            }
+        }
+        return mapped;
+    }
+    getSchema() {
+        return this.schema;
+    }
+    getPartitionSpec() {
+        return this.partitionSpec;
+    }
+}
+/**
+ * Multi-table transformer that routes WAL entries to table-specific Iceberg tables
+ */
+export class MultiTableTransformer {
+    transformers = new Map();
+    defaultTransformer;
+    walBucket;
+    icebergBucket;
+    doId;
+    state;
+    constructor(config) {
+        this.walBucket = config.walBucket;
+        this.icebergBucket = config.icebergBucket;
+        this.doId = config.doId;
+        this.state = config.state ?? null;
+        // Create default transformer for unmapped tables
+        const defaultConfig = {
+            walBucket: config.walBucket,
+            icebergBucket: config.icebergBucket,
+            doId: config.doId,
+            icebergTablePath: config.defaultTablePath,
+        };
+        if (config.state !== undefined) {
+            defaultConfig.state = config.state;
+        }
+        this.defaultTransformer = createWALIcebergTransformer(defaultConfig);
+    }
+    /**
+     * Register a table-specific transformer
+     */
+    registerTable(tableName, icebergTablePath) {
+        const transformerConfig = {
+            walBucket: this.walBucket,
+            icebergBucket: this.icebergBucket,
+            doId: this.doId,
+            icebergTablePath,
+        };
+        if (this.state !== null) {
+            transformerConfig.state = this.state;
+        }
+        const transformer = createWALIcebergTransformer(transformerConfig);
+        this.transformers.set(tableName, transformer);
+    }
+    /**
+     * Initialize all transformers
+     */
+    async initialize() {
+        await this.defaultTransformer.initialize();
+        for (const transformer of this.transformers.values()) {
+            await transformer.initialize();
+        }
+    }
+    /**
+     * Get transformer for a specific table
+     */
+    getTransformer(tableName) {
+        return this.transformers.get(tableName) ?? this.defaultTransformer;
+    }
+}
+//# sourceMappingURL=transformer.js.map