@dotdo/postgres 0.1.0

package/src/streaming/query-streamer.ts

@@ -0,0 +1,662 @@
+/**
+ * QueryStreamer - Cursor-based streaming for large query results
+ *
+ * @deprecated For new code, prefer LiveCDCStream for real-time CDC or direct
+ * query execution for one-time result sets. QueryStreamer SIMULATES cursors
+ * by buffering entire results, which doesn't provide true streaming.
+ *
+ * ## Why Use LiveCDCStream Instead?
+ *
+ * QueryStreamer has a fundamental limitation: it executes the full query
+ * upfront and buffers all results (see lines 362-364 in createCursor).
+ * This means:
+ *
+ * - Memory usage spikes when query executes, not when iterating
+ * - No true backpressure - results are already in memory
+ * - Not suitable for very large result sets
+ *
+ * LiveCDCStream uses PGLite's live.changes() API which:
+ *
+ * - Streams changes as they happen (true CDC)
+ * - Only buffers recent changes
+ * - Integrates directly with Iceberg ingestion
+ *
+ * ```typescript
+ * // RECOMMENDED: Use LiveCDCStream for real-time changes
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ * ```
+ *
+ * ## Legacy Usage (Backwards Compatibility)
+ *
+ * QueryStreamer is maintained for backwards compatibility with existing code:
+ *
+ * ```typescript
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 50 * 1024 * 1024, // 50MB
+ * })
+ *
+ * // Stream rows one at a time
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * ## Architecture
+ *
+ * QueryStreamer wraps a QueryExecutor and provides cursor-based iteration
+ * over query results. It tracks memory usage and supports backpressure
+ * to prevent OOM in memory-constrained environments like Cloudflare Workers.
+ *
+ * NOTE: In a real PGLite integration with true cursor support, this would
+ * use DECLARE CURSOR / FETCH SQL commands. Currently we simulate cursor
+ * behavior by buffering the result (see createCursor implementation).
+ *
+ * ## Memory Management
+ *
+ * - **batchSize**: Controls how many rows are fetched per cursor operation
+ * - **memoryLimit**: Hard limit that throws OOM error when exceeded
+ * - **highWaterMark**: Triggers pause/backpressure when exceeded
+ * - **lowWaterMark**: Resumes streaming when memory drops below this
+ *
+ * @module streaming/query-streamer
+ */
+
+// ============================================================================
+// Type Definitions
+// ============================================================================
+
+/**
+ * Generic row type for query results.
+ * Each key is a column name, value is the column value.
+ *
+ * Uses Record<string, unknown> per project convention for pure dictionary types.
+ * @see CLAUDE.md "Record vs Index Signature Usage"
+ */
+export type Row = Record<string, unknown>
+
+/**
+ * PostgreSQL field metadata from query result.
+ * Matches the pg wire protocol field description format.
+ */
+export interface FieldInfo {
+  /** Column name */
+  name: string
+  /** PostgreSQL data type OID */
+  dataTypeID: number
+  /** Table OID (0 if not from a table) */
+  tableID: number
+  /** Column number within the table (0 if not from a table) */
+  columnID: number
+  /** Data type size (-1 for variable length) */
+  dataTypeSize: number
+  /** Type-specific modifier (e.g., precision for numeric) */
+  dataTypeModifier: number
+  /** Format code (0 = text, 1 = binary) */
+  format: number
+}
+
+/**
+ * Configuration options for query streaming.
+ */
+export interface StreamOptions {
+  /** Number of rows to fetch per batch (default: 100) */
+  batchSize?: number
+  /** Name prefix for the database cursor (default: 'pglite_cursor') */
+  cursorName?: string
+  /** High water mark for backpressure in bytes (default: 10MB) */
+  highWaterMark?: number
+  /** Low water mark for resuming in bytes (default: 5MB) */
+  lowWaterMark?: number
+  /** Maximum memory limit in bytes (default: 50MB) */
+  memoryLimit?: number
+  /** Callback when memory pressure is detected */
+  onMemoryPressure?: () => void | Promise<void>
+  /** AbortSignal for cancellation support */
+  signal?: AbortSignal
+}
+
+/**
+ * Result from a cursor fetch operation with metadata.
+ */
+export interface CursorFetchResult<T extends Row = Row> {
+  /** The fetched rows */
+  rows: T[]
+  /** Whether more rows are available */
+  hasMore: boolean
+  /** Total number of rows fetched so far */
+  totalFetched: number
+}
+
+/**
+ * Cursor interface for streaming query results.
+ * Cursors support both explicit fetch() calls and async iteration.
+ */
+export interface Cursor<T extends Row = Row> extends AsyncIterable<T> {
+  /** Whether the cursor is still open */
+  readonly isOpen: boolean
+  /** Total number of rows fetched so far */
+  readonly totalFetched: number
+  /** Field metadata from the query */
+  readonly fields: FieldInfo[]
+  /** Fetch the next batch of rows */
+  fetch(count?: number): Promise<T[]>
+  /** Fetch with additional metadata */
+  fetchWithInfo(count?: number): Promise<CursorFetchResult<T>>
+  /** Close the cursor and release resources */
+  close(): Promise<void>
+}
+
+/**
+ * Minimal interface for query execution.
+ * This abstraction allows QueryStreamer to work with any database client
+ * that implements this interface.
+ */
+export interface QueryExecutor {
+  /** Execute a query and return results */
+  query<T extends Row = Row>(sql: string, params?: unknown[]): Promise<{ rows: T[]; fields?: FieldInfo[] }>
+  /** Execute a statement without returning results (optional) */
+  exec?(sql: string): Promise<void>
+}
+
+// ============================================================================
+// Utility Functions
+// ============================================================================
+
+/**
+ * Estimate memory usage of rows using JSON serialization.
+ * This is a rough approximation suitable for backpressure decisions.
+ *
+ * @param rows - Array of rows to estimate
+ * @returns Estimated memory usage in bytes
+ */
+export function estimateRowsMemory(rows: Row[]): number {
+  if (rows.length === 0) return 0
+  try {
+    return JSON.stringify(rows).length
+  } catch {
+    // Fallback: estimate 1KB per row for circular references or other issues
+    return rows.length * 1024
+  }
+}
+
+/**
+ * Custom error class for OOM conditions during streaming.
+ */
+export class StreamingOOMError extends Error {
+  constructor(message: string = 'Memory limit exceeded during streaming') {
+    super(`OOM: ${message}`)
+    this.name = 'StreamingOOMError'
+  }
+}
+
+/**
+ * Custom error class for aborted streaming operations.
+ */
+export class StreamingAbortedError extends Error {
+  constructor(message: string = 'Query streaming aborted') {
+    super(message)
+    this.name = 'StreamingAbortedError'
+  }
+}
+
+// ============================================================================
+// Imported Configuration
+// ============================================================================
+
+import {
+  DEFAULT_STREAMING_MEMORY_LIMIT,
+  DEFAULT_HIGH_WATER_MARK,
+  DEFAULT_LOW_WATER_MARK,
+  DEFAULT_BATCH_SIZE,
+  DEFAULT_CURSOR_PREFIX,
+} from '../config/streaming'
+
+// ============================================================================
+// Local Aliases (for backward compatibility)
+// ============================================================================
+
+/** Default memory limit (50MB) */
+const DEFAULT_MEMORY_LIMIT = DEFAULT_STREAMING_MEMORY_LIMIT
+
+// ============================================================================
+// Internal Types
+// ============================================================================
+
+/**
+ * Resolved options with all defaults applied
+ */
+interface ResolvedStreamOptions {
+  batchSize: number
+  cursorName: string
+  highWaterMark: number
+  lowWaterMark: number
+  memoryLimit: number
+  onMemoryPressure?: () => void | Promise<void>
+  signal?: AbortSignal
+}
+
+// ============================================================================
+// QueryStreamer Class
+// ============================================================================
+
+/**
+ * QueryStreamer provides cursor-based streaming for large query results.
+ *
+ * Uses PostgreSQL DECLARE CURSOR / FETCH to incrementally retrieve rows,
+ * preventing OOM by keeping memory usage bounded regardless of result set size.
+ *
+ * ## Thread Safety
+ *
+ * QueryStreamer maintains internal state (memory tracking, cursor registry)
+ * that is not thread-safe. In a Cloudflare Worker context, this is fine
+ * since each request runs in a single-threaded isolate.
+ *
+ * ## Resource Management
+ *
+ * Always ensure cursors are closed, either by:
+ * - Iterating to completion (automatic cleanup)
+ * - Calling cursor.close() explicitly
+ * - Calling streamer.closeAll() to cleanup all cursors
+ *
+ * @example Basic streaming
+ * ```typescript
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 10 * 1024 * 1024, // 10MB
+ * })
+ *
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * @example Cursor-based control
+ * ```typescript
+ * const cursor = await streamer.createCursor('SELECT * FROM large_table')
+ * try {
+ *   while (cursor.isOpen) {
+ *     const rows = await cursor.fetch(100)
+ *     if (rows.length === 0) break
+ *     await processRows(rows)
+ *   }
+ * } finally {
+ *   await cursor.close()
+ * }
+ * ```
+ *
+ * @example With abort signal
+ * ```typescript
+ * const controller = new AbortController()
+ * const streamer = new QueryStreamer(executor, { signal: controller.signal })
+ *
+ * setTimeout(() => controller.abort(), 5000) // Abort after 5s
+ *
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   await processRow(row)
+ * }
+ * ```
+ */
+export class QueryStreamer {
+  private readonly executor: QueryExecutor
+  private readonly options: ResolvedStreamOptions
+  private cursorCounter = 0
+  private readonly activeCursors = new Set<string>()
+  private memoryUsed = 0
+  private isPaused = false
+
+  /**
+   * Create a new QueryStreamer instance.
+   *
+   * @param executor - Database executor for running queries
+   * @param options - Streaming configuration options
+   */
+  constructor(executor: QueryExecutor, options: StreamOptions = {}) {
+    this.executor = executor
+    this.options = {
+      batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
+      cursorName: options.cursorName ?? DEFAULT_CURSOR_PREFIX,
+      highWaterMark: options.highWaterMark ?? DEFAULT_HIGH_WATER_MARK,
+      lowWaterMark: options.lowWaterMark ?? DEFAULT_LOW_WATER_MARK,
+      memoryLimit: options.memoryLimit ?? DEFAULT_MEMORY_LIMIT,
+      ...(options.onMemoryPressure !== undefined && { onMemoryPressure: options.onMemoryPressure }),
+      ...(options.signal !== undefined && { signal: options.signal }),
+    }
+
+    // Validate water marks
+    if (this.options.lowWaterMark >= this.options.highWaterMark) {
+      throw new Error('lowWaterMark must be less than highWaterMark')
+    }
+    if (this.options.highWaterMark >= this.options.memoryLimit) {
+      throw new Error('highWaterMark must be less than memoryLimit')
+    }
+  }
+
+  /**
+   * Generate a unique cursor name for this streaming session.
+   * @internal
+   */
+  private generateCursorName(): string {
+    this.cursorCounter++
+    return `${this.options.cursorName}_${this.cursorCounter}_${Date.now()}`
+  }
+
+  /**
+   * Check if abort signal has been triggered and throw if so.
+   * @internal
+   */
+  private checkAborted(): void {
+    if (this.options.signal?.aborted) {
+      throw new StreamingAbortedError()
+    }
+  }
+
+  /**
+   * Create a cursor for streaming query results.
+   *
+   * The cursor provides explicit control over result fetching, allowing
+   * fine-grained memory management and batch processing.
+   *
+   * @param sql - SQL query to execute
+   * @param params - Query parameters (optional)
+   * @returns A cursor for iterating over results
+   *
+   * @example
+   * ```typescript
+   * const cursor = await streamer.createCursor<User>('SELECT * FROM users WHERE active = $1', [true])
+   * try {
+   *   while (cursor.isOpen) {
+   *     const batch = await cursor.fetch(100)
+   *     if (batch.length === 0) break
+   *     await processBatch(batch)
+   *   }
+   * } finally {
+   *   await cursor.close()
+   * }
+   * ```
+   */
+  async createCursor<T extends Row = Row>(sql: string, params?: unknown[]): Promise<Cursor<T>> {
+    this.checkAborted()
+
+    const cursorName = this.generateCursorName()
+    let isOpen = true
+    let totalFetched = 0
+    let fields: FieldInfo[] = []
+
+    // Register cursor for tracking
+    this.activeCursors.add(cursorName)
+
+    // Execute the query to get fields info
+    // NOTE: In a real PGLite integration with true cursor support,
+    // this would use DECLARE CURSOR / FETCH SQL commands.
+    // Currently we simulate cursor behavior by buffering the result.
+    const initialResult = await this.executor.query<T>(sql, params)
+    fields = initialResult.fields || []
+
+    // Store rows for cursor iteration (simulated cursor)
+    let allRows = initialResult.rows
+    let currentPosition = 0
+
+    // Reference to parent for method access
+    const self = this
+
+    const cursor: Cursor<T> = {
+      get isOpen() {
+        return isOpen
+      },
+      get totalFetched() {
+        return totalFetched
+      },
+      get fields() {
+        return fields
+      },
+
+      fetch: async (count?: number): Promise<T[]> => {
+        if (!isOpen) {
+          return []
+        }
+
+        self.checkAborted()
+
+        const fetchCount = count ?? self.options.batchSize
+        const endPosition = Math.min(currentPosition + fetchCount, allRows.length)
+        const rows = allRows.slice(currentPosition, endPosition)
+        currentPosition = endPosition
+        totalFetched += rows.length
+
+        // Track memory usage
+        const rowsMemory = estimateRowsMemory(rows)
+        self.memoryUsed += rowsMemory
+
+        // Check hard memory limit
+        if (self.memoryUsed > self.options.memoryLimit) {
+          // Cleanup before throwing
+          isOpen = false
+          self.activeCursors.delete(cursorName)
+          allRows = []
+          throw new StreamingOOMError()
+        }
+
+        // Check if we need to pause (backpressure)
+        if (self.memoryUsed > self.options.highWaterMark) {
+          self.isPaused = true
+          if (self.options.onMemoryPressure) {
+            await self.options.onMemoryPressure()
+          }
+        }
+
+        // Close if no more rows
+        if (currentPosition >= allRows.length) {
+          isOpen = false
+        }
+
+        // Simulate memory release after rows are yielded to consumer
+        // In a real implementation, this would happen when rows go out of scope
+        self.memoryUsed = Math.max(0, self.memoryUsed - rowsMemory)
+
+        return rows
+      },
+
+      fetchWithInfo: async (count?: number): Promise<CursorFetchResult<T>> => {
+        const rows = await cursor.fetch(count)
+        return {
+          rows,
+          hasMore: isOpen,
+          totalFetched,
+        }
+      },
+
+      close: async (): Promise<void> => {
+        if (!isOpen && !self.activeCursors.has(cursorName)) return
+        isOpen = false
+        self.activeCursors.delete(cursorName)
+        // Clear references to allow GC
+        allRows = []
+        currentPosition = 0
+      },
+
+      [Symbol.asyncIterator]: async function* () {
+        try {
+          while (isOpen) {
+            const rows = await cursor.fetch()
+            if (rows.length === 0) break
+            for (const row of rows) {
+              yield row
+            }
+          }
+        } finally {
+          // Ensure cleanup even if iteration is interrupted
+          await cursor.close()
+        }
+      },
+    }
+
+    return cursor
+  }
+
+  /**
+   * Stream query results as an async iterator.
+   *
+   * This is the simplest way to process large result sets. Each row is
+   * yielded individually, with automatic cleanup when iteration completes.
+   *
+   * @param sql - SQL query to execute
+   * @param params - Query parameters (optional)
+   * @yields Individual rows from the query result
+   *
+   * @example
+   * ```typescript
+   * for await (const user of streamer.stream<User>('SELECT * FROM users')) {
+   *   await sendNotification(user.email)
+   * }
+   * ```
+   */
+  async *stream<T extends Row = Row>(sql: string, params?: unknown[]): AsyncGenerator<T> {
+    const cursor = await this.createCursor<T>(sql, params)
+    try {
+      for await (const row of cursor) {
+        yield row
+      }
+    } finally {
+      await cursor.close()
+    }
+  }
+
+  /**
+   * Stream query results in batches for bulk processing.
+   *
+   * Batch streaming is more efficient than row-by-row when doing
+   * bulk operations like inserting into another table or API calls.
+   *
+   * @param sql - SQL query to execute
+   * @param params - Query parameters (optional)
+   * @param batchSize - Number of rows per batch (defaults to options.batchSize)
+   * @yields Arrays of rows, each up to batchSize in length
+   *
+   * @example
+   * ```typescript
+   * for await (const batch of streamer.streamBatches<User>('SELECT * FROM users', [], 500)) {
+   *   await bulkInsert('users_backup', batch)
+   * }
+   * ```
+   */
+  async *streamBatches<T extends Row = Row>(
+    sql: string,
+    params?: unknown[],
+    batchSize?: number
+  ): AsyncGenerator<T[]> {
+    const cursor = await this.createCursor<T>(sql, params)
+    const fetchSize = batchSize ?? this.options.batchSize
+
+    try {
+      while (cursor.isOpen) {
+        const batch = await cursor.fetch(fetchSize)
+        if (batch.length === 0) break
+        yield batch
+      }
+    } finally {
+      await cursor.close()
+    }
+  }
+
+  // ============================================================================
+  // Memory and State Management
+  // ============================================================================
+
+  /**
+   * Get current estimated memory usage in bytes.
+   * This tracks memory allocated by active cursors.
+   */
+  getMemoryUsage(): number {
+    return this.memoryUsed
+  }
+
+  /**
+   * Check if currently under memory pressure.
+   * Returns true when memory usage exceeds the high water mark.
+   */
+  isUnderMemoryPressure(): boolean {
+    return this.memoryUsed > this.options.highWaterMark
+  }
+
+  /**
+   * Manually pause streaming (apply backpressure).
+   * New fetch operations will be blocked until resume() is called.
+   */
+  pause(): void {
+    this.isPaused = true
+  }
+
+  /**
+   * Resume streaming after a pause.
+   */
+  resume(): void {
+    this.isPaused = false
+  }
+
+  /**
+   * Check if streaming is currently paused.
+   */
+  get paused(): boolean {
+    return this.isPaused
+  }
+
+  /**
+   * Get the number of active cursors.
+   */
+  get activeCursorCount(): number {
+    return this.activeCursors.size
+  }
+
+  /**
+   * Close all active cursors and reset state.
+   * Call this to clean up resources when done with the streamer.
+   */
+  async closeAll(): Promise<void> {
+    this.activeCursors.clear()
+    this.memoryUsed = 0
+    this.isPaused = false
+  }
+}
+
+// ============================================================================
+// Convenience Functions
+// ============================================================================
+
+/**
+ * Convenience function to stream a query without managing a QueryStreamer instance.
+ *
+ * Creates a QueryStreamer internally and returns an async generator.
+ * Suitable for one-off streaming operations.
+ *
+ * @param executor - Query executor
+ * @param sql - SQL query
+ * @param params - Query parameters (optional)
+ * @param options - Streaming options (optional)
+ * @returns Async generator of rows
+ *
+ * @example
+ * ```typescript
+ * for await (const row of streamQuery(executor, 'SELECT * FROM logs WHERE date > $1', [yesterday])) {
+ *   console.log(row)
+ * }
+ * ```
+ */
+export function streamQuery<T extends Row = Row>(
+  executor: QueryExecutor,
+  sql: string,
+  params?: unknown[],
+  options?: StreamOptions
+): AsyncGenerator<T> {
+  const streamer = new QueryStreamer(executor, options)
+  return streamer.stream<T>(sql, params)
+}