@dotdo/postgres 0.1.0

package/dist/streaming/live-cdc-stream.js

@@ -0,0 +1,703 @@
+/**
+ * Live CDC Stream - Real CDC streaming using PGLite's live.changes()
+ *
+ * This module provides true CDC (Change Data Capture) streaming by wrapping
+ * PGLite's built-in live.changes() API. Unlike QueryStreamer which simulates
+ * cursors by buffering results, this provides real streaming of changes as
+ * they happen.
+ *
+ * ## Architecture
+ *
+ * PGLite's live.changes() works by:
+ * 1. Creating a temporary view of the query
+ * 2. Setting up triggers on underlying tables
+ * 3. Using PostgreSQL's LISTEN/NOTIFY for change detection
+ * 4. Computing diffs between state snapshots
+ *
+ * This module wraps that functionality and provides:
+ * - Async iterator interface for consuming changes
+ * - HTTP/SSE streaming integration
+ * - Integration with pglake's WAL-to-Iceberg transformer
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * // Consume changes as async iterator
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ *
+ * // Or get an SSE stream for HTTP response
+ * const response = new Response(cdc.toSSEStream(), {
+ *   headers: { 'Content-Type': 'text/event-stream' }
+ * })
+ * ```
+ *
+ * @module streaming/live-cdc-stream
+ */
+// ============================================================================
+// Default Configuration
+// ============================================================================
+const DEFAULT_BATCH_SIZE = 1;
+const DEFAULT_FLUSH_INTERVAL_MS = 0;
+const DEFAULT_HIGH_WATER_MARK = 1000;
+// ============================================================================
+// LiveCDCStream Class
+// ============================================================================
+/**
+ * LiveCDCStream provides real CDC streaming using PGLite's live.changes() API.
+ *
+ * This is the preferred way to stream changes from PostgreSQL in Durable Objects.
+ * It uses PGLite's built-in change detection mechanism which is more efficient
+ * than polling or simulated cursors.
+ *
+ * @example Basic usage
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * await stream.start()
+ *
+ * for await (const change of stream) {
+ *   console.log(`${change.__op__}: ${JSON.stringify(change)}`)
+ * }
+ * ```
+ *
+ * @example With filtering and transformation
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM orders WHERE status = $1',
+ *   params: ['pending'],
+ *   key: 'id',
+ *   filter: (change) => change.__op__ !== 'DELETE',
+ *   transform: (change) => ({
+ *     ...change,
+ *     processedAt: Date.now(),
+ *   }),
+ * })
+ * ```
+ */
+export class LiveCDCStream {
+    pg;
+    config;
+    handle = null;
+    buffer = [];
+    resolveWaiter = null;
+    aborted = false;
+    started = false;
+    stats;
+    lsnCounter = 0;
+    constructor(pg, config) {
+        this.pg = pg;
+        this.config = {
+            query: config.query,
+            params: config.params ?? null,
+            key: config.key,
+            batchSize: config.batchSize ?? DEFAULT_BATCH_SIZE,
+            flushIntervalMs: config.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS,
+            includeInitial: config.includeInitial ?? true,
+            highWaterMark: config.highWaterMark ?? DEFAULT_HIGH_WATER_MARK,
+            ...(config.signal !== undefined && { signal: config.signal }),
+            ...(config.transform !== undefined && { transform: config.transform }),
+            ...(config.filter !== undefined && { filter: config.filter }),
+        };
+        this.stats = {
+            totalChanges: 0,
+            insertCount: 0,
+            updateCount: 0,
+            deleteCount: 0,
+            resetCount: 0,
+            filteredCount: 0,
+            bufferSize: 0,
+            startedAt: 0,
+            lastChangeAt: null,
+            changesPerSecond: 0,
+            isActive: false,
+        };
+        // Handle abort signal
+        if (this.config.signal) {
+            this.config.signal.addEventListener('abort', () => {
+                this.abort();
+            }, { once: true });
+        }
+    }
+    /**
+     * Start the CDC stream.
+     *
+     * Must be called before iterating. Returns the initial changes if
+     * includeInitial is true (default).
+     */
+    async start() {
+        if (this.started) {
+            throw new Error('LiveCDCStream already started');
+        }
+        this.started = true;
+        this.stats.startedAt = Date.now();
+        this.stats.isActive = true;
+        // Handle pre-aborted signal
+        if (this.config.signal?.aborted) {
+            this.aborted = true;
+            this.stats.isActive = false;
+            return [];
+        }
+        const liveOptions = {
+            query: this.config.query,
+            params: this.config.params,
+            key: this.config.key,
+            callback: (changes) => this.handleChanges(changes),
+        };
+        if (this.config.signal !== undefined) {
+            liveOptions.signal = this.config.signal;
+        }
+        this.handle = await this.pg.live.changes(liveOptions);
+        // Process initial changes and add to buffer for stream consumption
+        const initialChanges = this.config.includeInitial
+            ? this.processChanges(this.handle.initialChanges)
+            : [];
+        // Add initial changes to buffer so streams can consume them
+        for (const change of initialChanges) {
+            this.buffer.push(change);
+        }
+        return initialChanges;
+    }
+    /**
+     * Stop the CDC stream and clean up resources.
+     */
+    async stop() {
+        if (!this.handle)
+            return;
+        this.stats.isActive = false;
+        this.aborted = true;
+        // Resolve any pending waiter with null to signal end
+        if (this.resolveWaiter) {
+            this.resolveWaiter(null);
+            this.resolveWaiter = null;
+        }
+        await this.handle.unsubscribe();
+        this.handle = null;
+    }
+    /**
+     * Manually trigger a refresh of the underlying query.
+     */
+    async refresh() {
+        if (!this.handle) {
+            throw new Error('LiveCDCStream not started');
+        }
+        await this.handle.refresh();
+    }
+    /**
+     * Get current stream statistics.
+     */
+    getStats() {
+        // Calculate changes per second
+        if (this.stats.startedAt > 0) {
+            const elapsedSeconds = (Date.now() - this.stats.startedAt) / 1000;
+            if (elapsedSeconds > 0) {
+                this.stats.changesPerSecond = this.stats.totalChanges / elapsedSeconds;
+            }
+        }
+        this.stats.bufferSize = this.buffer.length;
+        return { ...this.stats };
+    }
+    /**
+     * Get the field metadata from the query.
+     */
+    getFields() {
+        if (!this.handle) {
+            throw new Error('LiveCDCStream not started');
+        }
+        return this.handle.fields;
+    }
+    /**
+     * Check if stream is currently active.
+     */
+    get isActive() {
+        return this.stats.isActive && !this.aborted;
+    }
+    /**
+     * Async iterator implementation for consuming changes.
+     */
+    async *[Symbol.asyncIterator]() {
+        if (!this.started) {
+            await this.start();
+        }
+        while (!this.aborted) {
+            // First drain the buffer
+            while (this.buffer.length > 0) {
+                const change = this.buffer.shift();
+                if (change) {
+                    yield change;
+                }
+            }
+            // Wait for next change or abort
+            const change = await this.waitForChange();
+            if (change === null) {
+                break;
+            }
+            yield change;
+        }
+    }
+    /**
+     * Create a ReadableStream of CDC changes for HTTP streaming.
+     */
+    toReadableStream() {
+        const self = this;
+        let started = false;
+        return new ReadableStream({
+            async start() {
+                if (!started && !self.started) {
+                    await self.start();
+                    started = true;
+                }
+            },
+            async pull(controller) {
+                if (self.aborted) {
+                    controller.close();
+                    return;
+                }
+                // Drain buffer first
+                while (self.buffer.length > 0) {
+                    const change = self.buffer.shift();
+                    if (change) {
+                        controller.enqueue(change);
+                        return;
+                    }
+                }
+                // Wait for next change
+                const change = await self.waitForChange();
+                if (change === null) {
+                    controller.close();
+                }
+                else {
+                    controller.enqueue(change);
+                }
+            },
+            cancel() {
+                self.abort();
+            },
+        });
+    }
+    /**
+     * Create an SSE (Server-Sent Events) stream for HTTP streaming.
+     */
+    toSSEStream() {
+        const self = this;
+        const encoder = new TextEncoder();
+        let eventId = 0;
+        let started = false;
+        return new ReadableStream({
+            async start() {
+                if (!started && !self.started) {
+                    await self.start();
+                    started = true;
+                }
+            },
+            async pull(controller) {
+                if (self.aborted) {
+                    controller.enqueue(encoder.encode(`event: done\ndata: {}\n\n`));
+                    controller.close();
+                    return;
+                }
+                // Drain buffer first
+                while (self.buffer.length > 0) {
+                    const change = self.buffer.shift();
+                    if (change) {
+                        eventId++;
+                        const sseMessage = formatSSEMessage(change, eventId);
+                        controller.enqueue(encoder.encode(sseMessage));
+                        return;
+                    }
+                }
+                // Wait for next change
+                const change = await self.waitForChange();
+                if (change === null) {
+                    controller.enqueue(encoder.encode(`event: done\ndata: {}\n\n`));
+                    controller.close();
+                }
+                else {
+                    eventId++;
+                    const sseMessage = formatSSEMessage(change, eventId);
+                    controller.enqueue(encoder.encode(sseMessage));
+                }
+            },
+            cancel() {
+                self.abort();
+            },
+        });
+    }
+    /**
+     * Create an NDJSON stream for HTTP streaming.
+     */
+    toNDJSONStream() {
+        const self = this;
+        const encoder = new TextEncoder();
+        let started = false;
+        return new ReadableStream({
+            async start() {
+                if (!started && !self.started) {
+                    await self.start();
+                    started = true;
+                }
+            },
+            async pull(controller) {
+                if (self.aborted) {
+                    controller.close();
+                    return;
+                }
+                // Drain buffer first
+                while (self.buffer.length > 0) {
+                    const change = self.buffer.shift();
+                    if (change) {
+                        const line = JSON.stringify(change) + '\n';
+                        controller.enqueue(encoder.encode(line));
+                        return;
+                    }
+                }
+                // Wait for next change
+                const change = await self.waitForChange();
+                if (change === null) {
+                    controller.close();
+                }
+                else {
+                    const line = JSON.stringify(change) + '\n';
+                    controller.enqueue(encoder.encode(line));
+                }
+            },
+            cancel() {
+                self.abort();
+            },
+        });
+    }
+    /**
+     * Convert CDC changes to Iceberg-compatible events.
+     *
+     * This is used for integration with pglake's WAL-to-Iceberg transformer.
+     */
+    async *toIcebergEvents(options = {}) {
+        const schema = options.schema ?? 'public';
+        const table = options.table ?? 'unknown';
+        const shardId = options.shardId ?? 'cdc';
+        for await (const change of this) {
+            // Skip RESET events for Iceberg
+            if (change.__op__ === 'RESET') {
+                continue;
+            }
+            this.lsnCounter++;
+            const lsn = this.lsnCounter.toString().padStart(24, '0');
+            const event = {
+                lsn,
+                operation: change.__op__,
+                schema,
+                table,
+                changedColumns: change.__changed_columns__,
+                timestamp: Date.now(),
+                shardId,
+            };
+            // Extract primary key and row data
+            const rowData = { ...change };
+            delete rowData.__op__;
+            delete rowData.__changed_columns__;
+            delete rowData.__after__;
+            const keyValue = change[this.config.key];
+            if (keyValue !== undefined) {
+                event.primaryKey = { [this.config.key]: keyValue };
+            }
+            if (change.__op__ === 'INSERT') {
+                event.newRow = rowData;
+            }
+            else if (change.__op__ === 'UPDATE') {
+                event.newRow = rowData;
+                // For UPDATE, we could potentially track old values if needed
+            }
+            else if (change.__op__ === 'DELETE') {
+                event.oldRow = rowData;
+            }
+            yield event;
+        }
+    }
+    // ============================================================================
+    // Private Methods
+    // ============================================================================
+    handleChanges(changes) {
+        const processed = this.processChanges(changes);
+        for (const change of processed) {
+            // Check high water mark
+            if (this.buffer.length >= this.config.highWaterMark) {
+                // Drop oldest changes if buffer is full
+                this.buffer.shift();
+            }
+            this.buffer.push(change);
+            // If there's a waiter, resolve it immediately
+            if (this.resolveWaiter && this.buffer.length === 1) {
+                const nextChange = this.buffer.shift();
+                if (nextChange) {
+                    this.resolveWaiter(nextChange);
+                    this.resolveWaiter = null;
+                }
+            }
+        }
+    }
+    processChanges(changes) {
+        const result = [];
+        for (const change of changes) {
+            // Update stats
+            this.stats.totalChanges++;
+            this.stats.lastChangeAt = Date.now();
+            switch (change.__op__) {
+                case 'INSERT':
+                    this.stats.insertCount++;
+                    break;
+                case 'UPDATE':
+                    this.stats.updateCount++;
+                    break;
+                case 'DELETE':
+                    this.stats.deleteCount++;
+                    break;
+                case 'RESET':
+                    this.stats.resetCount++;
+                    break;
+            }
+            // Apply filter
+            if (this.config.filter && !this.config.filter(change)) {
+                this.stats.filteredCount++;
+                continue;
+            }
+            // Apply transform
+            let processed = change;
+            if (this.config.transform) {
+                processed = this.config.transform(change);
+                if (!processed) {
+                    this.stats.filteredCount++;
+                    continue;
+                }
+            }
+            result.push(processed);
+        }
+        return result;
+    }
+    waitForChange() {
+        if (this.aborted) {
+            return Promise.resolve(null);
+        }
+        // If buffer has items, return immediately
+        if (this.buffer.length > 0) {
+            return Promise.resolve(this.buffer.shift() ?? null);
+        }
+        // Wait for next change
+        return new Promise((resolve) => {
+            this.resolveWaiter = resolve;
+        });
+    }
+    abort() {
+        this.aborted = true;
+        this.stats.isActive = false;
+        if (this.resolveWaiter) {
+            this.resolveWaiter(null);
+            this.resolveWaiter = null;
+        }
+    }
+}
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Format a CDC change as an SSE message.
+ */
+function formatSSEMessage(change, eventId) {
+    const eventType = change.__op__.toLowerCase();
+    const data = JSON.stringify(change);
+    return `id: ${eventId}\nevent: ${eventType}\ndata: ${data}\n\n`;
+}
+/**
+ * Create a LiveCDCStream from a PGLite instance.
+ *
+ * Convenience function for creating a CDC stream.
+ *
+ * @example
+ * ```typescript
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * for await (const change of cdc) {
+ *   console.log(change)
+ * }
+ * ```
+ */
+export function createLiveCDCStream(pg, config) {
+    return new LiveCDCStream(pg, config);
+}
+/**
+ * Create a CDC stream that automatically converts to Iceberg events.
+ *
+ * This is a convenience wrapper for integrating with pglake's
+ * WAL-to-Iceberg transformer.
+ *
+ * @example
+ * ```typescript
+ * import { createIcebergCDCStream } from '@dotdo/postgres/streaming'
+ * import { WALToIcebergTransformer } from '@dotdo/pglake/ingest'
+ *
+ * const cdcEvents = createIcebergCDCStream(pg, {
+ *   query: 'SELECT * FROM orders',
+ *   key: 'id',
+ *   schema: 'public',
+ *   table: 'orders',
+ * })
+ *
+ * for await (const event of cdcEvents) {
+ *   await transformer.processEvent({
+ *     ...event,
+ *     columnTypes: { id: 'integer', ... }
+ *   })
+ * }
+ * ```
+ */
+export async function* createIcebergCDCStream(pg, config) {
+    const stream = new LiveCDCStream(pg, config);
+    await stream.start();
+    const eventConfig = {};
+    if (config.schema !== undefined) {
+        eventConfig.schema = config.schema;
+    }
+    if (config.table !== undefined) {
+        eventConfig.table = config.table;
+    }
+    if (config.shardId !== undefined) {
+        eventConfig.shardId = config.shardId;
+    }
+    yield* stream.toIcebergEvents(eventConfig);
+}
+/**
+ * Batch CDC changes into groups for efficient processing.
+ *
+ * @example
+ * ```typescript
+ * const batched = batchCDCChanges(cdcStream, {
+ *   maxSize: 100,
+ *   maxWaitMs: 1000,
+ * })
+ *
+ * for await (const batch of batched) {
+ *   await processChanges(batch)
+ * }
+ * ```
+ */
+export async function* batchCDCChanges(source, options) {
+    const { maxSize, maxWaitMs } = options;
+    let batch = [];
+    let timeoutId = null;
+    let flushPromiseResolve = null;
+    const flush = () => {
+        if (timeoutId) {
+            clearTimeout(timeoutId);
+            timeoutId = null;
+        }
+        if (batch.length === 0)
+            return null;
+        const result = batch;
+        batch = [];
+        return result;
+    };
+    const scheduleFlush = () => {
+        return new Promise((resolve) => {
+            flushPromiseResolve = resolve;
+            timeoutId = setTimeout(() => {
+                timeoutId = null;
+                if (flushPromiseResolve) {
+                    flushPromiseResolve();
+                    flushPromiseResolve = null;
+                }
+            }, maxWaitMs);
+        });
+    };
+    try {
+        for await (const change of source) {
+            batch.push(change);
+            if (batch.length >= maxSize) {
+                const result = flush();
+                if (result)
+                    yield result;
+            }
+            else if (batch.length === 1) {
+                // Start timeout on first item in batch
+                scheduleFlush().then(() => {
+                    // Timeout triggered, but we can't yield from here
+                    // The async iterator will check and flush on next iteration
+                });
+            }
+            // Check if timeout triggered flush
+            if (timeoutId === null && batch.length > 0) {
+                const result = flush();
+                if (result)
+                    yield result;
+            }
+        }
+        // Flush remaining items
+        const result = flush();
+        if (result)
+            yield result;
+    }
+    finally {
+        if (timeoutId) {
+            clearTimeout(timeoutId);
+        }
+    }
+}
+/**
+ * Merge multiple CDC streams into a single stream.
+ *
+ * Useful for monitoring changes across multiple tables.
+ *
+ * @example
+ * ```typescript
+ * const merged = mergeCDCStreams([
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM users', key: 'id' }),
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM orders', key: 'id' }),
+ * ])
+ *
+ * for await (const change of merged) {
+ *   console.log('Change from some table:', change)
+ * }
+ * ```
+ */
+export async function* mergeCDCStreams(streams) {
+    // Start all streams
+    await Promise.all(streams.map((s) => s.start()));
+    // Create merged async iterator using Promise.race
+    const iterators = streams.map((s, i) => ({
+        index: i,
+        iterator: s[Symbol.asyncIterator](),
+        done: false,
+        current: null,
+    }));
+    // Initial fetch from all iterators
+    await Promise.all(iterators.map(async (it) => {
+        it.current = await it.iterator.next();
+        it.done = it.current.done ?? false;
+    }));
+    while (iterators.some((it) => !it.done)) {
+        // Race for next available value
+        const activeIterators = iterators.filter((it) => !it.done && it.current && !it.current.done);
+        if (activeIterators.length === 0)
+            break;
+        for (const it of activeIterators) {
+            if (it.current && !it.current.done) {
+                yield { ...it.current.value, _sourceIndex: it.index };
+                it.current = await it.iterator.next();
+                it.done = it.current.done ?? false;
+            }
+        }
+    }
+}
+//# sourceMappingURL=live-cdc-stream.js.map