@dotdo/postgres 0.1.0

package/dist/worker/wal-r2.js

@@ -0,0 +1,689 @@
+/**
+ * R2 WAL Storage - Stream WAL entries to Cloudflare R2
+ * Task: postgres-54m - WAL streaming to R2 storage
+ *
+ * Implements durable WAL storage in R2 with:
+ * - Batched segment uploads organized by DO ID and time windows
+ * - zstd compression for storage efficiency
+ * - Checksum validation for data integrity
+ * - Retry logic for transient R2 failures
+ * - Configurable flush policies
+ */
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG = {
+    prefix: 'wal/',
+    compression: true,
+    timeWindowMs: 60 * 60 * 1000, // 1 hour
+    maxEntriesPerSegment: 10000,
+    maxSegmentBytes: 10 * 1024 * 1024, // 10MB
+    retryAttempts: 3,
+    retryBaseDelayMs: 100,
+    validateChecksums: true,
+};
+/**
+ * DO storage key for tracking last flushed LSN
+ */
+const LAST_FLUSHED_LSN_KEY = '__wal_last_flushed_lsn__';
+/**
+ * Calculate SHA-256 checksum of data
+ */
+async function calculateChecksum(data) {
+    const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
+}
+/**
+ * Compress data using gzip (available in Workers runtime)
+ */
+async function compressData(data) {
+    const stream = new CompressionStream('gzip');
+    const writer = stream.writable.getWriter();
+    const reader = stream.readable.getReader();
+    // Write data and close
+    writer.write(data);
+    writer.close();
+    // Read compressed result
+    const chunks = [];
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        chunks.push(value);
+    }
+    // Combine chunks
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
+/**
+ * Decompress gzip data
+ */
+async function decompressData(data) {
+    const stream = new DecompressionStream('gzip');
+    const writer = stream.writable.getWriter();
+    const reader = stream.readable.getReader();
+    // Write data and close
+    writer.write(data);
+    writer.close();
+    // Read decompressed result
+    const chunks = [];
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        chunks.push(value);
+    }
+    // Combine chunks
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
+/**
+ * Sleep for a given number of milliseconds
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Generate a segment key based on DO ID and time window
+ *
+ * Format: {prefix}{doId}/{year}/{month}/{day}/{hour}/{startLsn}-{endLsn}.wal
+ *
+ * This organization allows efficient range queries by time and enables
+ * easy cleanup of old segments based on retention policies.
+ */
+function generateSegmentKey(prefix, doId, windowStart, startLsn, endLsn) {
+    const date = new Date(windowStart);
+    const year = date.getUTCFullYear();
+    const month = String(date.getUTCMonth() + 1).padStart(2, '0');
+    const day = String(date.getUTCDate()).padStart(2, '0');
+    const hour = String(date.getUTCHours()).padStart(2, '0');
+    // Pad LSNs for proper lexicographic ordering
+    const paddedStartLsn = startLsn.padStart(20, '0');
+    const paddedEndLsn = endLsn.padStart(20, '0');
+    return `${prefix}${doId}/${year}/${month}/${day}/${hour}/${paddedStartLsn}-${paddedEndLsn}.wal`;
+}
+/**
+ * R2WALStorage - WAL storage implementation using Cloudflare R2
+ *
+ * Provides durable storage for WAL entries with:
+ * - Automatic batching into segments
+ * - Compression using gzip
+ * - Checksum validation
+ * - Retry logic for transient failures
+ * - Time-based organization for efficient queries
+ *
+ * @example
+ * ```typescript
+ * const storage = new R2WALStorage({
+ *   bucket: env.WAL_BUCKET,
+ *   doId: ctx.id.toString(),
+ *   state: ctx.storage,
+ * })
+ *
+ * // Use with WALManager
+ * const walManager = new WALManager(walConfig, storage)
+ * ```
+ */
+export class R2WALStorage {
+    // Note: bucket is package-private to allow access from WALRetentionManager
+    bucket;
+    doId;
+    prefix;
+    compression;
+    timeWindowMs;
+    maxEntriesPerSegment;
+    maxSegmentBytes;
+    retryAttempts;
+    retryBaseDelayMs;
+    validateChecksums;
+    state;
+    stats = {
+        segmentsUploaded: 0,
+        entriesStored: 0,
+        bytesWritten: 0,
+        bytesUncompressed: 0,
+        failedUploads: 0,
+        retryCount: 0,
+        lastUploadAt: null,
+        lastFlushedLsn: null,
+    };
+    // Internal buffer for batching entries before upload
+    pendingEntries = [];
+    pendingBytes = 0;
+    constructor(config) {
+        this.bucket = config.bucket;
+        this.doId = config.doId;
+        this.prefix = config.prefix ?? DEFAULT_CONFIG.prefix;
+        this.compression = config.compression ?? DEFAULT_CONFIG.compression;
+        this.timeWindowMs = config.timeWindowMs ?? DEFAULT_CONFIG.timeWindowMs;
+        this.maxEntriesPerSegment = config.maxEntriesPerSegment ?? DEFAULT_CONFIG.maxEntriesPerSegment;
+        this.maxSegmentBytes = config.maxSegmentBytes ?? DEFAULT_CONFIG.maxSegmentBytes;
+        this.retryAttempts = config.retryAttempts ?? DEFAULT_CONFIG.retryAttempts;
+        this.retryBaseDelayMs = config.retryBaseDelayMs ?? DEFAULT_CONFIG.retryBaseDelayMs;
+        this.validateChecksums = config.validateChecksums ?? DEFAULT_CONFIG.validateChecksums;
+        this.state = config.state ?? null;
+    }
+    /**
+     * Store WAL entries to R2
+     *
+     * Entries are batched into segments and uploaded when:
+     * - The segment reaches maxEntriesPerSegment
+     * - The segment size reaches maxSegmentBytes
+     * - This method is called (ensures entries are persisted)
+     */
+    async store(entries) {
+        if (entries.length === 0) {
+            return;
+        }
+        // Add entries to pending buffer
+        for (const entry of entries) {
+            const entrySize = JSON.stringify(entry).length;
+            this.pendingEntries.push(entry);
+            this.pendingBytes += entrySize;
+            // Check if we should flush
+            if (this.pendingEntries.length >= this.maxEntriesPerSegment ||
+                this.pendingBytes >= this.maxSegmentBytes) {
+                await this.flushPendingEntries();
+            }
+        }
+        // Always flush remaining entries to ensure durability
+        if (this.pendingEntries.length > 0) {
+            await this.flushPendingEntries();
+        }
+    }
+    /**
+     * Flush pending entries to R2 as a segment
+     */
+    async flushPendingEntries() {
+        if (this.pendingEntries.length === 0) {
+            return;
+        }
+        const entries = this.pendingEntries;
+        this.pendingEntries = [];
+        this.pendingBytes = 0;
+        const result = await this.uploadSegment(entries);
+        if (!result.success) {
+            // Re-add entries to pending buffer on failure
+            // They will be retried on next store() call
+            this.pendingEntries = entries.concat(this.pendingEntries);
+            this.pendingBytes = entries.reduce((sum, e) => sum + JSON.stringify(e).length, 0) + this.pendingBytes;
+            throw new Error(`Failed to upload WAL segment: ${result.error}`);
+        }
+        // Update last flushed LSN
+        if (result.metadata) {
+            this.stats.lastFlushedLsn = result.metadata.endLsn;
+            await this.persistLastFlushedLsn(result.metadata.endLsn);
+        }
+    }
+    /**
+     * Upload a segment of entries to R2
+     */
+    async uploadSegment(entries) {
+        const firstEntry = entries[0];
+        const lastEntry = entries[entries.length - 1];
+        if (!firstEntry || !lastEntry) {
+            return { success: true, retryAttempts: 0 };
+        }
+        const now = Date.now();
+        const windowStart = Math.floor(now / this.timeWindowMs) * this.timeWindowMs;
+        const windowEnd = windowStart + this.timeWindowMs;
+        const startLsn = firstEntry.lsn;
+        const endLsn = lastEntry.lsn;
+        // Serialize entries as newline-delimited JSON (NDJSON)
+        const jsonData = entries.map((e) => JSON.stringify(e)).join('\n');
+        const uncompressedData = new TextEncoder().encode(jsonData);
+        const uncompressedSize = uncompressedData.length;
+        // Calculate checksum before compression
+        const checksum = await calculateChecksum(uncompressedData);
+        // Compress if enabled
+        let finalData;
+        let compressionType = 'none';
+        if (this.compression) {
+            try {
+                finalData = await compressData(uncompressedData);
+                compressionType = 'gzip';
+            }
+            catch (error) {
+                // Fall back to uncompressed on compression failure
+                console.warn('[R2WALStorage] Compression failed, storing uncompressed:', error);
+                finalData = uncompressedData;
+            }
+        }
+        else {
+            finalData = uncompressedData;
+        }
+        const compressedSize = finalData.length;
+        // Build metadata
+        const metadata = {
+            doId: this.doId,
+            startLsn,
+            endLsn,
+            entryCount: entries.length,
+            uncompressedSize,
+            compressedSize,
+            compression: compressionType,
+            checksum,
+            createdAt: now,
+            windowStart,
+            windowEnd,
+        };
+        // Generate key
+        const key = generateSegmentKey(this.prefix, this.doId, windowStart, startLsn, endLsn);
+        // Upload with retry
+        let lastError = null;
+        let retryCount = 0;
+        for (let attempt = 0; attempt < this.retryAttempts; attempt++) {
+            try {
+                await this.bucket.put(key, finalData, {
+                    customMetadata: {
+                        ...Object.fromEntries(Object.entries(metadata).map(([k, v]) => [k, String(v)])),
+                    },
+                });
+                // Update stats
+                this.stats.segmentsUploaded++;
+                this.stats.entriesStored += entries.length;
+                this.stats.bytesWritten += compressedSize;
+                this.stats.bytesUncompressed += uncompressedSize;
+                this.stats.lastUploadAt = now;
+                this.stats.retryCount += retryCount;
+                return {
+                    success: true,
+                    key,
+                    metadata,
+                    retryAttempts: retryCount,
+                };
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                retryCount++;
+                this.stats.retryCount++;
+                if (attempt < this.retryAttempts - 1) {
+                    // Exponential backoff
+                    const delay = this.retryBaseDelayMs * Math.pow(2, attempt);
+                    await sleep(delay);
+                }
+            }
+        }
+        // All retries failed
+        this.stats.failedUploads++;
+        return {
+            success: false,
+            error: lastError?.message || 'Unknown error',
+            retryAttempts: retryCount,
+        };
+    }
+    /**
+     * Get the last stored LSN
+     *
+     * This reads from DO storage for fast access, falling back to
+     * scanning R2 if DO storage is not available.
+     */
+    async getLastLsn() {
+        // Try DO storage first (fast path)
+        if (this.state) {
+            const stored = await this.state.storage.get(LAST_FLUSHED_LSN_KEY);
+            if (stored) {
+                return BigInt(stored);
+            }
+        }
+        // Fall back to scanning R2 (slow path)
+        // List segments for this DO and find the latest
+        try {
+            const prefix = `${this.prefix}${this.doId}/`;
+            let cursor;
+            let latestLsn = null;
+            do {
+                const listOptions = { prefix, limit: 1000 };
+                if (cursor) {
+                    listOptions.cursor = cursor;
+                }
+                const result = await this.bucket.list(listOptions);
+                for (const obj of result.objects) {
+                    // Extract end LSN from key (format: .../startLsn-endLsn.wal)
+                    const match = obj.key.match(/(\d+)-(\d+)\.wal$/);
+                    if (match && match[2]) {
+                        const endLsn = BigInt(match[2]);
+                        if (latestLsn === null || endLsn > latestLsn) {
+                            latestLsn = endLsn;
+                        }
+                    }
+                }
+                cursor = result.truncated ? result.cursor : undefined;
+            } while (cursor);
+            // Cache in DO storage for next time
+            if (latestLsn !== null && this.state) {
+                await this.state.storage.put(LAST_FLUSHED_LSN_KEY, latestLsn.toString());
+            }
+            return latestLsn;
+        }
+        catch (error) {
+            console.error('[R2WALStorage] Failed to get last LSN from R2:', error);
+            return null;
+        }
+    }
+    /**
+     * Persist the last flushed LSN to DO storage
+     */
+    async persistLastFlushedLsn(lsn) {
+        if (this.state) {
+            await this.state.storage.put(LAST_FLUSHED_LSN_KEY, lsn);
+        }
+    }
+    /**
+     * Read entries from R2 storage starting from a given LSN
+     *
+     * This scans segments in order and returns entries from the specified LSN.
+     */
+    async readEntries(fromLsn, limit = 1000) {
+        const entries = [];
+        const prefix = `${this.prefix}${this.doId}/`;
+        try {
+            let cursor;
+            const segmentKeys = [];
+            // First, list all segments for this DO
+            do {
+                const listOptions = { prefix, limit: 1000 };
+                if (cursor) {
+                    listOptions.cursor = cursor;
+                }
+                const result = await this.bucket.list(listOptions);
+                for (const obj of result.objects) {
+                    // Check if this segment might contain entries >= fromLsn
+                    const match = obj.key.match(/(\d+)-(\d+)\.wal$/);
+                    if (match && match[2]) {
+                        const endLsn = BigInt(match[2]);
+                        if (endLsn >= fromLsn) {
+                            segmentKeys.push(obj.key);
+                        }
+                    }
+                }
+                cursor = result.truncated ? result.cursor : undefined;
+            } while (cursor);
+            // Sort segments by their start LSN
+            segmentKeys.sort();
+            // Read segments until we have enough entries
+            for (const key of segmentKeys) {
+                if (entries.length >= limit) {
+                    break;
+                }
+                const segmentEntries = await this.readSegment(key);
+                for (const entry of segmentEntries) {
+                    if (BigInt(entry.lsn) >= fromLsn) {
+                        entries.push(entry);
+                        if (entries.length >= limit) {
+                            break;
+                        }
+                    }
+                }
+            }
+            return entries;
+        }
+        catch (error) {
+            console.error('[R2WALStorage] Failed to read entries from R2:', error);
+            return [];
+        }
+    }
+    /**
+     * Read a single segment from R2
+     */
+    async readSegment(key) {
+        try {
+            const obj = await this.bucket.get(key);
+            if (!obj) {
+                return [];
+            }
+            const metadata = obj.customMetadata;
+            const compression = metadata?.compression || 'none';
+            let data = await obj.bytes();
+            // Decompress if needed
+            if (compression === 'gzip') {
+                data = await decompressData(data);
+            }
+            // Validate checksum if enabled
+            if (this.validateChecksums && metadata?.checksum) {
+                const calculatedChecksum = await calculateChecksum(data);
+                if (calculatedChecksum !== metadata.checksum) {
+                    throw new Error(`Checksum mismatch for segment ${key}`);
+                }
+            }
+            // Parse NDJSON
+            const jsonStr = new TextDecoder().decode(data);
+            const lines = jsonStr.split('\n').filter((line) => line.trim());
+            return lines.map((line) => JSON.parse(line));
+        }
+        catch (error) {
+            console.error(`[R2WALStorage] Failed to read segment ${key}:`, error);
+            return [];
+        }
+    }
+    /**
+     * Get storage statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * List all segment keys for this DO
+     */
+    async listSegments(options) {
+        const prefix = `${this.prefix}${this.doId}/`;
+        try {
+            const listOptions = { prefix, limit: options?.limit || 1000 };
+            if (options?.cursor) {
+                listOptions.cursor = options.cursor;
+            }
+            const result = await this.bucket.list(listOptions);
+            let keys = result.objects.map((obj) => obj.key);
+            // Filter by time if specified
+            if (options?.fromTime || options?.toTime) {
+                keys = keys.filter((key) => {
+                    // Extract date from key path
+                    const match = key.match(/\/(\d{4})\/(\d{2})\/(\d{2})\/(\d{2})\//);
+                    if (!match)
+                        return true;
+                    const year = match[1];
+                    const month = match[2];
+                    const day = match[3];
+                    const hour = match[4];
+                    if (!year || !month || !day || !hour)
+                        return true;
+                    const segmentTime = new Date(Date.UTC(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hour))).getTime();
+                    if (options.fromTime && segmentTime < options.fromTime)
+                        return false;
+                    if (options.toTime && segmentTime > options.toTime)
+                        return false;
+                    return true;
+                });
+            }
+            const response = {
+                keys,
+                truncated: result.truncated,
+            };
+            if (result.truncated && result.cursor) {
+                response.cursor = result.cursor;
+            }
+            return response;
+        }
+        catch (error) {
+            console.error('[R2WALStorage] Failed to list segments:', error);
+            return { keys: [], truncated: false };
+        }
+    }
+    /**
+     * Delete segments older than a given timestamp
+     *
+     * Useful for implementing retention policies.
+     */
+    async deleteSegmentsBefore(timestamp) {
+        let deletedCount = 0;
+        const keysToDelete = [];
+        // List all segments
+        let cursor;
+        do {
+            const listOptions = {};
+            if (cursor) {
+                listOptions.cursor = cursor;
+            }
+            const result = await this.listSegments(Object.keys(listOptions).length > 0 ? listOptions : undefined);
+            for (const key of result.keys) {
+                // Extract date from key path
+                const match = key.match(/\/(\d{4})\/(\d{2})\/(\d{2})\/(\d{2})\//);
+                if (match) {
+                    const year = match[1];
+                    const month = match[2];
+                    const day = match[3];
+                    const hour = match[4];
+                    if (year && month && day && hour) {
+                        const segmentTime = new Date(Date.UTC(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hour))).getTime();
+                        if (segmentTime < timestamp) {
+                            keysToDelete.push(key);
+                        }
+                    }
+                }
+            }
+            cursor = result.cursor;
+        } while (cursor);
+        // Delete in batches
+        const batchSize = 100;
+        for (let i = 0; i < keysToDelete.length; i += batchSize) {
+            const batch = keysToDelete.slice(i, i + batchSize);
+            try {
+                await this.bucket.delete(batch);
+                deletedCount += batch.length;
+            }
+            catch (error) {
+                console.error('[R2WALStorage] Failed to delete segments:', error);
+            }
+        }
+        return deletedCount;
+    }
+    /**
+     * Get the compression ratio (compressed/uncompressed)
+     */
+    getCompressionRatio() {
+        if (this.stats.bytesUncompressed === 0) {
+            return 1;
+        }
+        return this.stats.bytesWritten / this.stats.bytesUncompressed;
+    }
+    /**
+     * Get the R2 bucket (for use by retention manager)
+     */
+    getBucket() {
+        return this.bucket;
+    }
+    /**
+     * Get the DO ID
+     */
+    getDoId() {
+        return this.doId;
+    }
+    /**
+     * Get the key prefix
+     */
+    getPrefix() {
+        return this.prefix;
+    }
+    /**
+     * Delete multiple segments by key
+     * Used by retention manager for cleanup
+     */
+    async deleteSegments(keys) {
+        if (keys.length === 0) {
+            return 0;
+        }
+        let deletedCount = 0;
+        const batchSize = 100;
+        for (let i = 0; i < keys.length; i += batchSize) {
+            const batch = keys.slice(i, i + batchSize);
+            try {
+                await this.bucket.delete(batch);
+                deletedCount += batch.length;
+            }
+            catch (error) {
+                console.error('[R2WALStorage] Failed to delete segments:', error);
+            }
+        }
+        return deletedCount;
+    }
+    /**
+     * Get detailed segment information including metadata
+     * Used by retention manager for cleanup decisions
+     */
+    async getSegmentInfo(key) {
+        try {
+            const obj = await this.bucket.head(key);
+            if (!obj) {
+                return null;
+            }
+            const metadata = obj.customMetadata;
+            return {
+                key,
+                size: obj.size,
+                createdAt: metadata?.createdAt ? parseInt(metadata.createdAt) : obj.uploaded.getTime(),
+                startLsn: metadata?.startLsn ? BigInt(metadata.startLsn) : 0n,
+                endLsn: metadata?.endLsn ? BigInt(metadata.endLsn) : 0n,
+                entryCount: metadata?.entryCount ? parseInt(metadata.entryCount) : 0,
+            };
+        }
+        catch (error) {
+            console.error(`[R2WALStorage] Failed to get segment info for ${key}:`, error);
+            return null;
+        }
+    }
+    /**
+     * List segments with detailed metadata
+     * Used by retention manager for cleanup decisions
+     */
+    async listSegmentsWithInfo(options) {
+        const result = await this.listSegments(options);
+        const segments = [];
+        let totalSize = 0;
+        // Get detailed info for each segment
+        for (const key of result.keys) {
+            const info = await this.getSegmentInfo(key);
+            if (info) {
+                segments.push(info);
+                totalSize += info.size;
+            }
+        }
+        const response = {
+            segments,
+            truncated: result.truncated,
+            totalSize,
+        };
+        if (result.cursor !== undefined) {
+            response.cursor = result.cursor;
+        }
+        return response;
+    }
+}
+/**
+ * Create an R2WALStorage instance
+ *
+ * @example
+ * ```typescript
+ * const storage = createR2WALStorage({
+ *   bucket: env.WAL_BUCKET,
+ *   doId: ctx.id.toString(),
+ *   state: ctx.storage,
+ * })
+ * ```
+ */
+export function createR2WALStorage(config) {
+    return new R2WALStorage(config);
+}
+//# sourceMappingURL=wal-r2.js.map