@dotdo/postgres 0.1.0

package/dist/streaming/query-streamer.d.ts

@@ -0,0 +1,379 @@
+/**
+ * QueryStreamer - Cursor-based streaming for large query results
+ *
+ * @deprecated For new code, prefer LiveCDCStream for real-time CDC or direct
+ * query execution for one-time result sets. QueryStreamer SIMULATES cursors
+ * by buffering entire results, which doesn't provide true streaming.
+ *
+ * ## Why Use LiveCDCStream Instead?
+ *
+ * QueryStreamer has a fundamental limitation: it executes the full query
+ * upfront and buffers all results (see lines 362-364 in createCursor).
+ * This means:
+ *
+ * - Memory usage spikes when query executes, not when iterating
+ * - No true backpressure - results are already in memory
+ * - Not suitable for very large result sets
+ *
+ * LiveCDCStream uses PGLite's live.changes() API which:
+ *
+ * - Streams changes as they happen (true CDC)
+ * - Only buffers recent changes
+ * - Integrates directly with Iceberg ingestion
+ *
+ * ```typescript
+ * // RECOMMENDED: Use LiveCDCStream for real-time changes
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ * ```
+ *
+ * ## Legacy Usage (Backwards Compatibility)
+ *
+ * QueryStreamer is maintained for backwards compatibility with existing code:
+ *
+ * ```typescript
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 50 * 1024 * 1024, // 50MB
+ * })
+ *
+ * // Stream rows one at a time
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * ## Architecture
+ *
+ * QueryStreamer wraps a QueryExecutor and provides cursor-based iteration
+ * over query results. It tracks memory usage and supports backpressure
+ * to prevent OOM in memory-constrained environments like Cloudflare Workers.
+ *
+ * NOTE: In a real PGLite integration with true cursor support, this would
+ * use DECLARE CURSOR / FETCH SQL commands. Currently we simulate cursor
+ * behavior by buffering the result (see createCursor implementation).
+ *
+ * ## Memory Management
+ *
+ * - **batchSize**: Controls how many rows are fetched per cursor operation
+ * - **memoryLimit**: Hard limit that throws OOM error when exceeded
+ * - **highWaterMark**: Triggers pause/backpressure when exceeded
+ * - **lowWaterMark**: Resumes streaming when memory drops below this
+ *
+ * @module streaming/query-streamer
+ */
+/**
+ * Generic row type for query results.
+ * Each key is a column name, value is the column value.
+ *
+ * Uses Record<string, unknown> per project convention for pure dictionary types.
+ * @see CLAUDE.md "Record vs Index Signature Usage"
+ */
+export type Row = Record<string, unknown>;
+/**
+ * PostgreSQL field metadata from query result.
+ * Matches the pg wire protocol field description format.
+ */
+export interface FieldInfo {
+    /** Column name */
+    name: string;
+    /** PostgreSQL data type OID */
+    dataTypeID: number;
+    /** Table OID (0 if not from a table) */
+    tableID: number;
+    /** Column number within the table (0 if not from a table) */
+    columnID: number;
+    /** Data type size (-1 for variable length) */
+    dataTypeSize: number;
+    /** Type-specific modifier (e.g., precision for numeric) */
+    dataTypeModifier: number;
+    /** Format code (0 = text, 1 = binary) */
+    format: number;
+}
+/**
+ * Configuration options for query streaming.
+ */
+export interface StreamOptions {
+    /** Number of rows to fetch per batch (default: 100) */
+    batchSize?: number;
+    /** Name prefix for the database cursor (default: 'pglite_cursor') */
+    cursorName?: string;
+    /** High water mark for backpressure in bytes (default: 10MB) */
+    highWaterMark?: number;
+    /** Low water mark for resuming in bytes (default: 5MB) */
+    lowWaterMark?: number;
+    /** Maximum memory limit in bytes (default: 50MB) */
+    memoryLimit?: number;
+    /** Callback when memory pressure is detected */
+    onMemoryPressure?: () => void | Promise<void>;
+    /** AbortSignal for cancellation support */
+    signal?: AbortSignal;
+}
+/**
+ * Result from a cursor fetch operation with metadata.
+ */
+export interface CursorFetchResult<T extends Row = Row> {
+    /** The fetched rows */
+    rows: T[];
+    /** Whether more rows are available */
+    hasMore: boolean;
+    /** Total number of rows fetched so far */
+    totalFetched: number;
+}
+/**
+ * Cursor interface for streaming query results.
+ * Cursors support both explicit fetch() calls and async iteration.
+ */
+export interface Cursor<T extends Row = Row> extends AsyncIterable<T> {
+    /** Whether the cursor is still open */
+    readonly isOpen: boolean;
+    /** Total number of rows fetched so far */
+    readonly totalFetched: number;
+    /** Field metadata from the query */
+    readonly fields: FieldInfo[];
+    /** Fetch the next batch of rows */
+    fetch(count?: number): Promise<T[]>;
+    /** Fetch with additional metadata */
+    fetchWithInfo(count?: number): Promise<CursorFetchResult<T>>;
+    /** Close the cursor and release resources */
+    close(): Promise<void>;
+}
+/**
+ * Minimal interface for query execution.
+ * This abstraction allows QueryStreamer to work with any database client
+ * that implements this interface.
+ */
+export interface QueryExecutor {
+    /** Execute a query and return results */
+    query<T extends Row = Row>(sql: string, params?: unknown[]): Promise<{
+        rows: T[];
+        fields?: FieldInfo[];
+    }>;
+    /** Execute a statement without returning results (optional) */
+    exec?(sql: string): Promise<void>;
+}
+/**
+ * Estimate memory usage of rows using JSON serialization.
+ * This is a rough approximation suitable for backpressure decisions.
+ *
+ * @param rows - Array of rows to estimate
+ * @returns Estimated memory usage in bytes
+ */
+export declare function estimateRowsMemory(rows: Row[]): number;
+/**
+ * Custom error class for OOM conditions during streaming.
+ */
+export declare class StreamingOOMError extends Error {
+    constructor(message?: string);
+}
+/**
+ * Custom error class for aborted streaming operations.
+ */
+export declare class StreamingAbortedError extends Error {
+    constructor(message?: string);
+}
+/**
+ * QueryStreamer provides cursor-based streaming for large query results.
+ *
+ * Uses PostgreSQL DECLARE CURSOR / FETCH to incrementally retrieve rows,
+ * preventing OOM by keeping memory usage bounded regardless of result set size.
+ *
+ * ## Thread Safety
+ *
+ * QueryStreamer maintains internal state (memory tracking, cursor registry)
+ * that is not thread-safe. In a Cloudflare Worker context, this is fine
+ * since each request runs in a single-threaded isolate.
+ *
+ * ## Resource Management
+ *
+ * Always ensure cursors are closed, either by:
+ * - Iterating to completion (automatic cleanup)
+ * - Calling cursor.close() explicitly
+ * - Calling streamer.closeAll() to cleanup all cursors
+ *
+ * @example Basic streaming
+ * ```typescript
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 10 * 1024 * 1024, // 10MB
+ * })
+ *
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * @example Cursor-based control
+ * ```typescript
+ * const cursor = await streamer.createCursor('SELECT * FROM large_table')
+ * try {
+ *   while (cursor.isOpen) {
+ *     const rows = await cursor.fetch(100)
+ *     if (rows.length === 0) break
+ *     await processRows(rows)
+ *   }
+ * } finally {
+ *   await cursor.close()
+ * }
+ * ```
+ *
+ * @example With abort signal
+ * ```typescript
+ * const controller = new AbortController()
+ * const streamer = new QueryStreamer(executor, { signal: controller.signal })
+ *
+ * setTimeout(() => controller.abort(), 5000) // Abort after 5s
+ *
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   await processRow(row)
+ * }
+ * ```
+ */
+export declare class QueryStreamer {
+    private readonly executor;
+    private readonly options;
+    private cursorCounter;
+    private readonly activeCursors;
+    private memoryUsed;
+    private isPaused;
+    /**
+     * Create a new QueryStreamer instance.
+     *
+     * @param executor - Database executor for running queries
+     * @param options - Streaming configuration options
+     */
+    constructor(executor: QueryExecutor, options?: StreamOptions);
+    /**
+     * Generate a unique cursor name for this streaming session.
+     * @internal
+     */
+    private generateCursorName;
+    /**
+     * Check if abort signal has been triggered and throw if so.
+     * @internal
+     */
+    private checkAborted;
+    /**
+     * Create a cursor for streaming query results.
+     *
+     * The cursor provides explicit control over result fetching, allowing
+     * fine-grained memory management and batch processing.
+     *
+     * @param sql - SQL query to execute
+     * @param params - Query parameters (optional)
+     * @returns A cursor for iterating over results
+     *
+     * @example
+     * ```typescript
+     * const cursor = await streamer.createCursor<User>('SELECT * FROM users WHERE active = $1', [true])
+     * try {
+     *   while (cursor.isOpen) {
+     *     const batch = await cursor.fetch(100)
+     *     if (batch.length === 0) break
+     *     await processBatch(batch)
+     *   }
+     * } finally {
+     *   await cursor.close()
+     * }
+     * ```
+     */
+    createCursor<T extends Row = Row>(sql: string, params?: unknown[]): Promise<Cursor<T>>;
+    /**
+     * Stream query results as an async iterator.
+     *
+     * This is the simplest way to process large result sets. Each row is
+     * yielded individually, with automatic cleanup when iteration completes.
+     *
+     * @param sql - SQL query to execute
+     * @param params - Query parameters (optional)
+     * @yields Individual rows from the query result
+     *
+     * @example
+     * ```typescript
+     * for await (const user of streamer.stream<User>('SELECT * FROM users')) {
+     *   await sendNotification(user.email)
+     * }
+     * ```
+     */
+    stream<T extends Row = Row>(sql: string, params?: unknown[]): AsyncGenerator<T>;
+    /**
+     * Stream query results in batches for bulk processing.
+     *
+     * Batch streaming is more efficient than row-by-row when doing
+     * bulk operations like inserting into another table or API calls.
+     *
+     * @param sql - SQL query to execute
+     * @param params - Query parameters (optional)
+     * @param batchSize - Number of rows per batch (defaults to options.batchSize)
+     * @yields Arrays of rows, each up to batchSize in length
+     *
+     * @example
+     * ```typescript
+     * for await (const batch of streamer.streamBatches<User>('SELECT * FROM users', [], 500)) {
+     *   await bulkInsert('users_backup', batch)
+     * }
+     * ```
+     */
+    streamBatches<T extends Row = Row>(sql: string, params?: unknown[], batchSize?: number): AsyncGenerator<T[]>;
+    /**
+     * Get current estimated memory usage in bytes.
+     * This tracks memory allocated by active cursors.
+     */
+    getMemoryUsage(): number;
+    /**
+     * Check if currently under memory pressure.
+     * Returns true when memory usage exceeds the high water mark.
+     */
+    isUnderMemoryPressure(): boolean;
+    /**
+     * Manually pause streaming (apply backpressure).
+     * New fetch operations will be blocked until resume() is called.
+     */
+    pause(): void;
+    /**
+     * Resume streaming after a pause.
+     */
+    resume(): void;
+    /**
+     * Check if streaming is currently paused.
+     */
+    get paused(): boolean;
+    /**
+     * Get the number of active cursors.
+     */
+    get activeCursorCount(): number;
+    /**
+     * Close all active cursors and reset state.
+     * Call this to clean up resources when done with the streamer.
+     */
+    closeAll(): Promise<void>;
+}
+/**
+ * Convenience function to stream a query without managing a QueryStreamer instance.
+ *
+ * Creates a QueryStreamer internally and returns an async generator.
+ * Suitable for one-off streaming operations.
+ *
+ * @param executor - Query executor
+ * @param sql - SQL query
+ * @param params - Query parameters (optional)
+ * @param options - Streaming options (optional)
+ * @returns Async generator of rows
+ *
+ * @example
+ * ```typescript
+ * for await (const row of streamQuery(executor, 'SELECT * FROM logs WHERE date > $1', [yesterday])) {
+ *   console.log(row)
+ * }
+ * ```
+ */
+export declare function streamQuery<T extends Row = Row>(executor: QueryExecutor, sql: string, params?: unknown[], options?: StreamOptions): AsyncGenerator<T>;
+//# sourceMappingURL=query-streamer.d.ts.map

package/dist/streaming/query-streamer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-streamer.d.ts","sourceRoot":"","sources":["../../src/streaming/query-streamer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuEG;AAMH;;;;;;GAMG;AACH,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAEzC;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,wCAAwC;IACxC,OAAO,EAAE,MAAM,CAAA;IACf,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,CAAA;IAChB,8CAA8C;IAC9C,YAAY,EAAE,MAAM,CAAA;IACpB,2DAA2D;IAC3D,gBAAgB,EAAE,MAAM,CAAA;IACxB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,oDAAoD;IACpD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC7C,2CAA2C;IAC3C,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG;IACpD,uBAAuB;IACvB,IAAI,EAAE,CAAC,EAAE,CAAA;IACT,sCAAsC;IACtC,OAAO,EAAE,OAAO,CAAA;IAChB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAA;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,MAAM,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IACnE,uCAAuC;IACvC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,CAAA;IAC5B,mCAAmC;IACnC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;IACnC,qCAAqC;IACrC,aAAa,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5D,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,yCAAyC;IACzC,KAAK,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC,CAAA;IACzG,+DAA+D;IAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAClC;AAMD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,CAQtD;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,GAAE,MAAiD;CAIvE;AAED;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,OAAO,GAAE,MAAkC;CAIxD;AA0CD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,aAAa,CAAI;IACzB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAClD,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,QAAQ,CAAQ;IAExB;;;;;OAKG;gBACS,QAAQ,EAAE,aAAa,EAAE,OAAO,GAAE,aAAkB;IAqBhE;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAK1B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAMpB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,YAAY,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAuH5F;;;;;;;;;;;;;;;;OAgBG;IACI,MAAM,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC;IAWtF;;;;;;;;;;;;;;;;;OAiBG;IACI,aAAa,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,EACtC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,SAAS,CAAC,EAAE,MAAM,GACjB,cAAc,CAAC,CAAC,EAAE,CAAC;IAmBtB;;;OAGG;IACH,cAAc,IAAI,MAAM;IAIxB;;;OAGG;IACH,qBAAqB,IAAI,OAAO;IAIhC;;;OAGG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,MAAM,IAAI,IAAI;IAId;;OAEG;IACH,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAKhC;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,WAAW,CAAC,CAAC,SAAS,GAAG,GAAG,GAAG,EAC7C,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,cAAc,CAAC,CAAC,CAAC,CAGnB"}