@dotdo/postgres 0.1.0

package/dist/streaming/cdc-iceberg-connector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdc-iceberg-connector.js","sourceRoot":"","sources":["../../src/streaming/cdc-iceberg-connector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,EACL,aAAa,GAGd,MAAM,mBAAmB,CAAA;AAuH1B,+EAA+E;AAC/E,wBAAwB;AACxB,+EAA+E;AAE/E,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAC/B,MAAM,yBAAyB,GAAG,IAAI,CAAA;AAEtC,+EAA+E;AAC/E,yBAAyB;AACzB,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAO,gBAAgB;IACV,EAAE,CAAgB;IAClB,WAAW,CAAoB;IAC/B,MAAM,CAAkB;IACxB,MAAM,CAOtB;IAEO,OAAO,GAAwD,IAAI,GAAG,EAAE,CAAA;IACxE,WAAW,GAA8E,EAAE,CAAA;IAC3F,UAAU,GAAG,CAAC,CAAA;IACd,UAAU,GAA0C,IAAI,CAAA;IACxD,OAAO,GAAG,KAAK,CAAA;IACf,QAAQ,GAAG,KAAK,CAAA;IAEhB,OAAO,GAAqB;QAClC,oBAAoB,EAAE,CAAC;QACvB,aAAa,EAAE,IAAI,GAAG,EAAE;QACxB,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,IAAI;QACjB,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;KACjB,CAAA;IAED,YAAY,MAA8B;QACxC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAA;QACnB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAA;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE;YAC7C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,kBAAkB;YACjD,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,yBAAyB;YACpE,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7D,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;YAChE,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;SACvE,CAAA;QAED,sBAAsB;QACtB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBAChD,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAClC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QACrD,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAEnC,yBAAyB;QACzB,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;QAEnC,oCAAoC;QACpC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,GAAG,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,EAAE,CAAA;YAE9D,MAAM,MAAM,GAAG,IAAI,aAAa,CAA0B,IAAI,CAAC,EAAE,EAAE;gBACjE,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,GAAG,EAAE,WAAW,CAAC,GAAG;gBACpB,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC;gBACvE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;aACxE,CAAC,CAAA;YAEF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YACnC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;YAE5C,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YACpC,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAA;QAE9C,6BAA6B;QAC7B,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3B,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;gBAClC,CAAC,CAAC,CAAA;YACJ,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAA;QAE/B,yBAAyB;QACzB,IAAI,CAAC,aAAa,EAAE,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAM;QAE1C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAA;QAEpB,mBAAmB;QACnB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QACxB,CAAC;QAED,mBAAmB;QACnB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;QACrF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QAE/B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;QACpB,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,CAAC,CAAA;QAE9B,yBAAyB;QACzB,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;QACpB,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAA;QAEjC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;QACpB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO;gBACL,eAAe,EAAE,CAAC;gBAClB,YAAY,EAAE,EAAE;gBAChB,gBAAgB,EAAE,EAAE;gBACpB,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,EAAE;aACX,CAAA;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAA;QAErB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAE7C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;QAC7B,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACrC,IAAI,CAAC,qBAAqB,EAAE,CAAA;QAC5B,IAAI,CAAC,aAAa,EAAE,CAAA;QAEpB,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAA;IAChF,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAA;IACvC,CAAC;IAED,+EAA+E;IAC/E,kBAAkB;IAClB,+EAA+E;IAEvE,KAAK,CAAC,aAAa,CACzB,MAA8C,EAC9C,WAA2B;QAE3B,MAAM,SAAS,GAAG,GAAG,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,EAAE,CAAA;QAE9D,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YAEpB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;gBAClC,IAAI,IAAI,CAAC,QAAQ;oBAAE,MAAK;gBAExB,oBAAoB;gBACpB,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;oBAAE,SAAQ;gBAEvC,uBAAuB;gBACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;gBAE5D,gBAAgB;gBAChB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAE/B,iBAAiB;gBACjB,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAA;gBACnC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBACjE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAA;gBAEzD,0CAA0C;gBAC1C,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;gBAE7C,6BAA6B;gBAC7B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;oBACrD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,CAAC,KAAc,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAEO,iBAAiB,CACvB,MAA0C,EAC1C,WAA2B;QAE3B,IAAI,CAAC,UAAU,EAAE,CAAA;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QAExD,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,MAAM,EAA6B,CAAA;QACxD,OAAO,OAAO,CAAC,MAAM,CAAA;QACrB,OAAO,OAAO,CAAC,mBAAmB,CAAA;QAClC,OAAO,OAAO,CAAC,SAAS,CAAA;QAExB,MAAM,KAAK,GAAuE;YAChF,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,MAAM,CAAC,MAAwC;YAC1D,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,GAAG,CAAC,WAAW,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC;SACvF,CAAA;QAED,sBAAsB;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QACzC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAA;QACpD,CAAC;QAED,kCAAkC;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;YACtB,iDAAiD;YACjD,4DAA4D;QAC9D,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;QACxB,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,WAAW,CAAC,KAAY,EAAE,OAAe;QAC/C,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAE1B,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QACrC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,OAAO,GAAG,EAAE,KAAK,CAAC,CAAA;QACjE,CAAC;IACH,CAAC;IAEO,qBAAqB;QAC3B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAA;YACnE,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,GAAG,cAAc,CAAA;YACnF,CAAC;QACH,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QAC1C,CAAC;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,uBAAuB,CACrC,EAAkB,EAClB,WAA+B,EAC/B,WAA2B,EAC3B,OAAuE;IAEvE,OAAO,IAAI,gBAAgB,CAAC;QAC1B,EAAE;QACF,WAAW;QACX,MAAM,EAAE,CAAC,WAAW,CAAC;QACrB,GAAG,OAAO;KACX,CAAC,CAAA;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,4BAA4B,CACjD,EAAkB,EAClB,MAA6C;IAE7C,MAAM,MAAM,GAAG,IAAI,aAAa,CAA0B,EAAE,EAAE;QAC5D,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;KAC9D,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAA;IACpD,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IAEpB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;YAAE,SAAQ;QAEvC,UAAU,EAAE,CAAA;QACZ,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QAEnD,MAAM,OAAO,GAAG,EAAE,GAAG,MAAM,EAA6B,CAAA;QACxD,OAAO,OAAO,CAAC,MAAM,CAAA;QACrB,OAAO,OAAO,CAAC,mBAAmB,CAAA;QAClC,OAAO,OAAO,CAAC,SAAS,CAAA;QAExB,MAAM,KAAK,GAAuE;YAChF,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO;YACP,SAAS,EAAE,MAAM,CAAC,MAAwC;YAC1D,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;SAC7E,CAAA;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAA;QAC/C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;QACxB,CAAC;QAED,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC"}

package/dist/streaming/index.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Streaming Module - Query result streaming with OOM prevention
+ *
+ * This module provides cursor-based streaming for large query results,
+ * preventing OOM by keeping memory usage bounded regardless of result set size.
+ *
+ * ## Overview
+ *
+ * When processing large query results in memory-constrained environments like
+ * Cloudflare Workers (128MB limit), traditional approaches that buffer entire
+ * result sets can cause OOM errors. This module provides streaming alternatives
+ * that process rows incrementally.
+ *
+ * ## Key Components
+ *
+ * - **LiveCDCStream**: Real CDC streaming using PGLite's live.changes() (RECOMMENDED)
+ * - **CDCIcebergBridge**: Connect CDC streams to Iceberg ingestion
+ * - **QueryStreamer**: Legacy cursor-based streaming (backwards compatibility)
+ * - **MemoryBoundedStream**: Generic stream wrapper with configurable memory limits
+ * - **BackpressureController**: Producer/consumer flow control
+ * - **Response Streaming**: HTTP response utilities (NDJSON, SSE, JSON arrays)
+ *
+ * ## CDC Streaming (Recommended for New Code)
+ *
+ * For real-time change data capture, use LiveCDCStream which wraps PGLite's
+ * built-in live.changes() API:
+ *
+ * ```typescript
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * // Create a CDC stream
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * // Stream changes as they happen
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ *
+ * // Or create an SSE response for HTTP streaming
+ * const response = new Response(cdc.toSSEStream(), {
+ *   headers: { 'Content-Type': 'text/event-stream' }
+ * })
+ * ```
+ *
+ * ## Iceberg Integration
+ *
+ * To stream CDC changes to Iceberg tables:
+ *
+ * ```typescript
+ * import { CDCIcebergBridge } from '@dotdo/postgres/streaming'
+ *
+ * const bridge = new CDCIcebergBridge({
+ *   pg,
+ *   transformer,
+ *   tables: [
+ *     { query: 'SELECT * FROM users', key: 'id', schema: 'public', table: 'users' },
+ *   ],
+ * })
+ *
+ * await bridge.start()
+ * ```
+ *
+ * ## Legacy Query Streaming
+ *
+ * QueryStreamer simulates cursor-based streaming by buffering entire results.
+ * For new code, prefer LiveCDCStream for real-time changes or direct query
+ * execution for one-time result sets.
+ *
+ * ```typescript
+ * import { QueryStreamer, createNDJSONStream } from '@dotdo/postgres/streaming'
+ *
+ * // Legacy: Create a streamer with memory limits
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 10 * 1024 * 1024, // 10MB
+ * })
+ *
+ * // Stream rows one at a time
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * ## Memory Management
+ *
+ * The streaming components use a water mark system for memory management:
+ *
+ * - **memoryLimit**: Hard limit that throws OOM error when exceeded
+ * - **highWaterMark**: Triggers pause/backpressure when exceeded
+ * - **lowWaterMark**: Resumes streaming when memory drops below this
+ *
+ * The gap between high and low water marks prevents thrashing (rapid
+ * pause/resume cycles).
+ *
+ * @module streaming
+ */
+export { QueryStreamer, streamQuery, type Row, type FieldInfo, type StreamOptions, type Cursor, type CursorFetchResult, type QueryExecutor, } from './query-streamer';
+export { MemoryBoundedStream, createMemoryBoundedStream, type MemoryBoundedStreamConfig, type MemoryStats, } from './memory-bounded-stream';
+/**
+ * Alias for MemoryStats for backwards compatibility
+ */
+export type { MemoryStats as MemoryMetrics } from './memory-bounded-stream';
+export { BackpressureController, createBackpressureController, type BackpressureConfig, type BackpressureStats, type BackpressureMetrics, } from './backpressure-controller';
+export { createReadableStream, createNDJSONStream, createSSEStream, createJSONArrayStream, createStreamResult, pipeThrough, filter, take, skip, type ReadableStreamOptions, type NDJSONStreamOptions, type StreamResult, } from './response-streaming';
+export { BufferPool, RingBuffer, BatchAccumulator, estimateSize, estimateRowsSize, OBJECT_BASE_OVERHEAD, BYTES_PER_CHAR, STRING_HEADER_OVERHEAD, ARRAY_BASE_OVERHEAD, ARRAY_ELEMENT_OVERHEAD, type BufferPoolConfig, type BufferPoolStats, type RingBufferConfig, type BatchAccumulatorConfig, } from './buffer-pool';
+export { LiveCDCStream, createLiveCDCStream, createIcebergCDCStream, batchCDCChanges, mergeCDCStreams, type CDCOperationType, type CDCChangeBase, type CDCInsertChange, type CDCUpdateChange, type CDCDeleteChange, type CDCResetChange, type CDCChange, type PGLiteWithLive, type LiveNamespace, type LiveChangesOptions, type LiveChangesHandle, type LiveCDCStreamConfig, type CDCStreamStats, type IcebergCDCEvent, } from './live-cdc-stream';
+export { CDCIcebergBridge, createSingleTableBridge, createSimpleCDCIcebergStream, type WALEventData, type WALEntry, type IcebergTransformer, type TransformationResult, type CDCTableConfig, type CDCIcebergBridgeConfig, type CDCBridgeMetrics, } from './cdc-iceberg-connector';
+//# sourceMappingURL=index.d.ts.map

package/dist/streaming/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/streaming/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkGG;AAMH,OAAO,EACL,aAAa,EACb,WAAW,EACX,KAAK,GAAG,EACR,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,MAAM,EACX,KAAK,iBAAiB,EACtB,KAAK,aAAa,GACnB,MAAM,kBAAkB,CAAA;AAMzB,OAAO,EACL,mBAAmB,EACnB,yBAAyB,EACzB,KAAK,yBAAyB,EAC9B,KAAK,WAAW,GACjB,MAAM,yBAAyB,CAAA;AAEhC;;GAEG;AACH,YAAY,EAAE,WAAW,IAAI,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAM3E,OAAO,EACL,sBAAsB,EACtB,4BAA4B,EAC5B,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,GACzB,MAAM,2BAA2B,CAAA;AAMlC,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,kBAAkB,EAClB,WAAW,EACX,MAAM,EACN,IAAI,EACJ,IAAI,EACJ,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,YAAY,GAClB,MAAM,sBAAsB,CAAA;AAM7B,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,GAC5B,MAAM,eAAe,CAAA;AAMtB,OAAO,EACL,aAAa,EACb,mBAAmB,EACnB,sBAAsB,EACtB,eAAe,EACf,eAAe,EACf,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,eAAe,GACrB,MAAM,mBAAmB,CAAA;AAM1B,OAAO,EACL,gBAAgB,EAChB,uBAAuB,EACvB,4BAA4B,EAC5B,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,GACtB,MAAM,yBAAyB,CAAA"}

package/dist/streaming/index.js

@@ -0,0 +1,128 @@
+/**
+ * Streaming Module - Query result streaming with OOM prevention
+ *
+ * This module provides cursor-based streaming for large query results,
+ * preventing OOM by keeping memory usage bounded regardless of result set size.
+ *
+ * ## Overview
+ *
+ * When processing large query results in memory-constrained environments like
+ * Cloudflare Workers (128MB limit), traditional approaches that buffer entire
+ * result sets can cause OOM errors. This module provides streaming alternatives
+ * that process rows incrementally.
+ *
+ * ## Key Components
+ *
+ * - **LiveCDCStream**: Real CDC streaming using PGLite's live.changes() (RECOMMENDED)
+ * - **CDCIcebergBridge**: Connect CDC streams to Iceberg ingestion
+ * - **QueryStreamer**: Legacy cursor-based streaming (backwards compatibility)
+ * - **MemoryBoundedStream**: Generic stream wrapper with configurable memory limits
+ * - **BackpressureController**: Producer/consumer flow control
+ * - **Response Streaming**: HTTP response utilities (NDJSON, SSE, JSON arrays)
+ *
+ * ## CDC Streaming (Recommended for New Code)
+ *
+ * For real-time change data capture, use LiveCDCStream which wraps PGLite's
+ * built-in live.changes() API:
+ *
+ * ```typescript
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * // Create a CDC stream
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * // Stream changes as they happen
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ *
+ * // Or create an SSE response for HTTP streaming
+ * const response = new Response(cdc.toSSEStream(), {
+ *   headers: { 'Content-Type': 'text/event-stream' }
+ * })
+ * ```
+ *
+ * ## Iceberg Integration
+ *
+ * To stream CDC changes to Iceberg tables:
+ *
+ * ```typescript
+ * import { CDCIcebergBridge } from '@dotdo/postgres/streaming'
+ *
+ * const bridge = new CDCIcebergBridge({
+ *   pg,
+ *   transformer,
+ *   tables: [
+ *     { query: 'SELECT * FROM users', key: 'id', schema: 'public', table: 'users' },
+ *   ],
+ * })
+ *
+ * await bridge.start()
+ * ```
+ *
+ * ## Legacy Query Streaming
+ *
+ * QueryStreamer simulates cursor-based streaming by buffering entire results.
+ * For new code, prefer LiveCDCStream for real-time changes or direct query
+ * execution for one-time result sets.
+ *
+ * ```typescript
+ * import { QueryStreamer, createNDJSONStream } from '@dotdo/postgres/streaming'
+ *
+ * // Legacy: Create a streamer with memory limits
+ * const streamer = new QueryStreamer(executor, {
+ *   batchSize: 100,
+ *   memoryLimit: 10 * 1024 * 1024, // 10MB
+ * })
+ *
+ * // Stream rows one at a time
+ * for await (const row of streamer.stream('SELECT * FROM large_table')) {
+ *   console.log(row)
+ * }
+ * ```
+ *
+ * ## Memory Management
+ *
+ * The streaming components use a water mark system for memory management:
+ *
+ * - **memoryLimit**: Hard limit that throws OOM error when exceeded
+ * - **highWaterMark**: Triggers pause/backpressure when exceeded
+ * - **lowWaterMark**: Resumes streaming when memory drops below this
+ *
+ * The gap between high and low water marks prevents thrashing (rapid
+ * pause/resume cycles).
+ *
+ * @module streaming
+ */
+// ============================================================================
+// Query Streaming
+// ============================================================================
+export { QueryStreamer, streamQuery, } from './query-streamer';
+// ============================================================================
+// Memory-Bounded Streaming
+// ============================================================================
+export { MemoryBoundedStream, createMemoryBoundedStream, } from './memory-bounded-stream';
+// ============================================================================
+// Backpressure Control
+// ============================================================================
+export { BackpressureController, createBackpressureController, } from './backpressure-controller';
+// ============================================================================
+// HTTP Response Streaming
+// ============================================================================
+export { createReadableStream, createNDJSONStream, createSSEStream, createJSONArrayStream, createStreamResult, pipeThrough, filter, take, skip, } from './response-streaming';
+// ============================================================================
+// Buffer Management Utilities
+// ============================================================================
+export { BufferPool, RingBuffer, BatchAccumulator, estimateSize, estimateRowsSize, OBJECT_BASE_OVERHEAD, BYTES_PER_CHAR, STRING_HEADER_OVERHEAD, ARRAY_BASE_OVERHEAD, ARRAY_ELEMENT_OVERHEAD, } from './buffer-pool';
+// ============================================================================
+// Live CDC Streaming (RECOMMENDED for new code)
+// ============================================================================
+export { LiveCDCStream, createLiveCDCStream, createIcebergCDCStream, batchCDCChanges, mergeCDCStreams, } from './live-cdc-stream';
+// ============================================================================
+// CDC to Iceberg Connector
+// ============================================================================
+export { CDCIcebergBridge, createSingleTableBridge, createSimpleCDCIcebergStream, } from './cdc-iceberg-connector';
+//# sourceMappingURL=index.js.map

package/dist/streaming/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/streaming/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkGG;AAEH,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E,OAAO,EACL,aAAa,EACb,WAAW,GAOZ,MAAM,kBAAkB,CAAA;AAEzB,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E,OAAO,EACL,mBAAmB,EACnB,yBAAyB,GAG1B,MAAM,yBAAyB,CAAA;AAOhC,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E,OAAO,EACL,sBAAsB,EACtB,4BAA4B,GAI7B,MAAM,2BAA2B,CAAA;AAElC,+EAA+E;AAC/E,0BAA0B;AAC1B,+EAA+E;AAE/E,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,kBAAkB,EAClB,WAAW,EACX,MAAM,EACN,IAAI,EACJ,IAAI,GAIL,MAAM,sBAAsB,CAAA;AAE7B,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GAKvB,MAAM,eAAe,CAAA;AAEtB,+EAA+E;AAC/E,gDAAgD;AAChD,+EAA+E;AAE/E,OAAO,EACL,aAAa,EACb,mBAAmB,EACnB,sBAAsB,EACtB,eAAe,EACf,eAAe,GAehB,MAAM,mBAAmB,CAAA;AAE1B,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E,OAAO,EACL,gBAAgB,EAChB,uBAAuB,EACvB,4BAA4B,GAQ7B,MAAM,yBAAyB,CAAA"}

package/dist/streaming/live-cdc-stream.d.ts

@@ -0,0 +1,400 @@
+/**
+ * Live CDC Stream - Real CDC streaming using PGLite's live.changes()
+ *
+ * This module provides true CDC (Change Data Capture) streaming by wrapping
+ * PGLite's built-in live.changes() API. Unlike QueryStreamer which simulates
+ * cursors by buffering results, this provides real streaming of changes as
+ * they happen.
+ *
+ * ## Architecture
+ *
+ * PGLite's live.changes() works by:
+ * 1. Creating a temporary view of the query
+ * 2. Setting up triggers on underlying tables
+ * 3. Using PostgreSQL's LISTEN/NOTIFY for change detection
+ * 4. Computing diffs between state snapshots
+ *
+ * This module wraps that functionality and provides:
+ * - Async iterator interface for consuming changes
+ * - HTTP/SSE streaming integration
+ * - Integration with pglake's WAL-to-Iceberg transformer
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { createLiveCDCStream } from '@dotdo/postgres/streaming'
+ *
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * // Consume changes as async iterator
+ * for await (const change of cdc) {
+ *   console.log(change.__op__, change)
+ * }
+ *
+ * // Or get an SSE stream for HTTP response
+ * const response = new Response(cdc.toSSEStream(), {
+ *   headers: { 'Content-Type': 'text/event-stream' }
+ * })
+ * ```
+ *
+ * @module streaming/live-cdc-stream
+ */
+/**
+ * CDC operation type matching PGLite's live.changes() operations
+ */
+export type CDCOperationType = 'INSERT' | 'UPDATE' | 'DELETE' | 'RESET';
+/**
+ * Base change event interface
+ */
+export interface CDCChangeBase {
+    /** Operation type */
+    __op__: CDCOperationType;
+    /** Columns that changed (for UPDATE) */
+    __changed_columns__: string[];
+    /** Position hint for ordering */
+    __after__?: number;
+}
+/**
+ * INSERT change event
+ */
+export interface CDCInsertChange<_T = Record<string, unknown>> extends CDCChangeBase {
+    __op__: 'INSERT';
+    __after__: number;
+}
+/**
+ * UPDATE change event
+ */
+export interface CDCUpdateChange<_T = Record<string, unknown>> extends CDCChangeBase {
+    __op__: 'UPDATE';
+    __after__: number;
+}
+/**
+ * DELETE change event
+ */
+export interface CDCDeleteChange<_T = Record<string, unknown>> extends Omit<CDCChangeBase, '__after__'> {
+    __op__: 'DELETE';
+}
+/**
+ * RESET change event (full table refresh needed)
+ */
+export interface CDCResetChange extends CDCChangeBase {
+    __op__: 'RESET';
+    __changed_columns__: [];
+}
+/**
+ * Union type for all change events
+ */
+export type CDCChange<T = Record<string, unknown>> = (CDCInsertChange<T> & T) | (CDCUpdateChange<T> & T) | (CDCDeleteChange<T> & T) | CDCResetChange;
+/**
+ * PGLite interface with live extension
+ */
+export interface PGLiteWithLive {
+    live: LiveNamespace;
+}
+/**
+ * LiveNamespace from PGLite
+ */
+export interface LiveNamespace {
+    changes<T = Record<string, unknown>>(query: string | LiveChangesOptions<T>, params?: unknown[] | null, key?: string, callback?: (changes: CDCChange<T>[]) => void): Promise<LiveChangesHandle<T>>;
+}
+/**
+ * Options for live.changes()
+ */
+export interface LiveChangesOptions<T = Record<string, unknown>> {
+    query: string;
+    params?: unknown[] | null;
+    key: string;
+    callback?: (changes: CDCChange<T>[]) => void;
+    signal?: AbortSignal;
+}
+/**
+ * Handle returned by live.changes()
+ */
+export interface LiveChangesHandle<T = Record<string, unknown>> {
+    fields: Array<{
+        name: string;
+        dataTypeID: number;
+    }>;
+    initialChanges: CDCChange<T>[];
+    subscribe: (callback: (changes: CDCChange<T>[]) => void) => void;
+    unsubscribe: (callback?: (changes: CDCChange<T>[]) => void) => Promise<void>;
+    refresh: () => Promise<void>;
+}
+/**
+ * Configuration for LiveCDCStream
+ */
+export interface LiveCDCStreamConfig<T = Record<string, unknown>> {
+    /** SQL query to watch */
+    query: string;
+    /** Query parameters */
+    params?: unknown[] | null;
+    /** Primary key column for change tracking (required) */
+    key: string;
+    /** AbortSignal for cancellation */
+    signal?: AbortSignal;
+    /** Batch size for buffering changes (default: 1) */
+    batchSize?: number;
+    /** Flush interval for batched changes in ms (default: 0 = immediate) */
+    flushIntervalMs?: number;
+    /** Transform function applied to each change */
+    transform?: (change: CDCChange<T>) => CDCChange<T> | null;
+    /** Filter function to selectively process changes */
+    filter?: (change: CDCChange<T>) => boolean;
+    /** Whether to include initial data on startup (default: true) */
+    includeInitial?: boolean;
+    /** High water mark for buffering (default: 1000 changes) */
+    highWaterMark?: number;
+}
+/**
+ * CDC stream statistics
+ */
+export interface CDCStreamStats {
+    /** Total changes processed */
+    totalChanges: number;
+    /** Inserts processed */
+    insertCount: number;
+    /** Updates processed */
+    updateCount: number;
+    /** Deletes processed */
+    deleteCount: number;
+    /** Resets processed */
+    resetCount: number;
+    /** Changes filtered out */
+    filteredCount: number;
+    /** Current buffer size */
+    bufferSize: number;
+    /** Stream start time */
+    startedAt: number;
+    /** Last change timestamp */
+    lastChangeAt: number | null;
+    /** Changes per second (rolling average) */
+    changesPerSecond: number;
+    /** Whether stream is active */
+    isActive: boolean;
+}
+/**
+ * CDC event for Iceberg integration
+ */
+export interface IcebergCDCEvent<T = Record<string, unknown>> {
+    /** Log sequence number (monotonic) */
+    lsn: string;
+    /** Operation type */
+    operation: 'INSERT' | 'UPDATE' | 'DELETE';
+    /** Schema name */
+    schema: string;
+    /** Table name */
+    table: string;
+    /** Primary key values */
+    primaryKey?: Record<string, unknown>;
+    /** New row data (for INSERT/UPDATE) */
+    newRow?: T;
+    /** Old row data (for UPDATE/DELETE) */
+    oldRow?: T;
+    /** Changed columns (for UPDATE) */
+    changedColumns: string[];
+    /** Timestamp */
+    timestamp: number;
+    /** Source shard ID */
+    shardId: string;
+}
+/**
+ * LiveCDCStream provides real CDC streaming using PGLite's live.changes() API.
+ *
+ * This is the preferred way to stream changes from PostgreSQL in Durable Objects.
+ * It uses PGLite's built-in change detection mechanism which is more efficient
+ * than polling or simulated cursors.
+ *
+ * @example Basic usage
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * await stream.start()
+ *
+ * for await (const change of stream) {
+ *   console.log(`${change.__op__}: ${JSON.stringify(change)}`)
+ * }
+ * ```
+ *
+ * @example With filtering and transformation
+ * ```typescript
+ * const stream = new LiveCDCStream(pg, {
+ *   query: 'SELECT * FROM orders WHERE status = $1',
+ *   params: ['pending'],
+ *   key: 'id',
+ *   filter: (change) => change.__op__ !== 'DELETE',
+ *   transform: (change) => ({
+ *     ...change,
+ *     processedAt: Date.now(),
+ *   }),
+ * })
+ * ```
+ */
+export declare class LiveCDCStream<T = Record<string, unknown>> implements AsyncIterable<CDCChange<T>> {
+    private readonly pg;
+    private readonly config;
+    private handle;
+    private buffer;
+    private resolveWaiter;
+    private aborted;
+    private started;
+    private stats;
+    private lsnCounter;
+    constructor(pg: PGLiteWithLive, config: LiveCDCStreamConfig<T>);
+    /**
+     * Start the CDC stream.
+     *
+     * Must be called before iterating. Returns the initial changes if
+     * includeInitial is true (default).
+     */
+    start(): Promise<CDCChange<T>[]>;
+    /**
+     * Stop the CDC stream and clean up resources.
+     */
+    stop(): Promise<void>;
+    /**
+     * Manually trigger a refresh of the underlying query.
+     */
+    refresh(): Promise<void>;
+    /**
+     * Get current stream statistics.
+     */
+    getStats(): CDCStreamStats;
+    /**
+     * Get the field metadata from the query.
+     */
+    getFields(): Array<{
+        name: string;
+        dataTypeID: number;
+    }>;
+    /**
+     * Check if stream is currently active.
+     */
+    get isActive(): boolean;
+    /**
+     * Async iterator implementation for consuming changes.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<CDCChange<T>, void, undefined>;
+    /**
+     * Create a ReadableStream of CDC changes for HTTP streaming.
+     */
+    toReadableStream(): ReadableStream<CDCChange<T>>;
+    /**
+     * Create an SSE (Server-Sent Events) stream for HTTP streaming.
+     */
+    toSSEStream(): ReadableStream<Uint8Array>;
+    /**
+     * Create an NDJSON stream for HTTP streaming.
+     */
+    toNDJSONStream(): ReadableStream<Uint8Array>;
+    /**
+     * Convert CDC changes to Iceberg-compatible events.
+     *
+     * This is used for integration with pglake's WAL-to-Iceberg transformer.
+     */
+    toIcebergEvents(options?: {
+        schema?: string;
+        table?: string;
+        shardId?: string;
+    }): AsyncGenerator<IcebergCDCEvent<T>, void, undefined>;
+    private handleChanges;
+    private processChanges;
+    private waitForChange;
+    private abort;
+}
+/**
+ * Create a LiveCDCStream from a PGLite instance.
+ *
+ * Convenience function for creating a CDC stream.
+ *
+ * @example
+ * ```typescript
+ * const cdc = createLiveCDCStream(pg, {
+ *   query: 'SELECT * FROM users',
+ *   key: 'id',
+ * })
+ *
+ * for await (const change of cdc) {
+ *   console.log(change)
+ * }
+ * ```
+ */
+export declare function createLiveCDCStream<T = Record<string, unknown>>(pg: PGLiteWithLive, config: LiveCDCStreamConfig<T>): LiveCDCStream<T>;
+/**
+ * Create a CDC stream that automatically converts to Iceberg events.
+ *
+ * This is a convenience wrapper for integrating with pglake's
+ * WAL-to-Iceberg transformer.
+ *
+ * @example
+ * ```typescript
+ * import { createIcebergCDCStream } from '@dotdo/postgres/streaming'
+ * import { WALToIcebergTransformer } from '@dotdo/pglake/ingest'
+ *
+ * const cdcEvents = createIcebergCDCStream(pg, {
+ *   query: 'SELECT * FROM orders',
+ *   key: 'id',
+ *   schema: 'public',
+ *   table: 'orders',
+ * })
+ *
+ * for await (const event of cdcEvents) {
+ *   await transformer.processEvent({
+ *     ...event,
+ *     columnTypes: { id: 'integer', ... }
+ *   })
+ * }
+ * ```
+ */
+export declare function createIcebergCDCStream<T = Record<string, unknown>>(pg: PGLiteWithLive, config: LiveCDCStreamConfig<T> & {
+    schema?: string;
+    table?: string;
+    shardId?: string;
+}): AsyncGenerator<IcebergCDCEvent<T>, void, undefined>;
+/**
+ * Batch CDC changes into groups for efficient processing.
+ *
+ * @example
+ * ```typescript
+ * const batched = batchCDCChanges(cdcStream, {
+ *   maxSize: 100,
+ *   maxWaitMs: 1000,
+ * })
+ *
+ * for await (const batch of batched) {
+ *   await processChanges(batch)
+ * }
+ * ```
+ */
+export declare function batchCDCChanges<T = Record<string, unknown>>(source: AsyncIterable<CDCChange<T>>, options: {
+    /** Maximum batch size */
+    maxSize: number;
+    /** Maximum wait time in ms before flushing partial batch */
+    maxWaitMs: number;
+}): AsyncGenerator<CDCChange<T>[], void, undefined>;
+/**
+ * Merge multiple CDC streams into a single stream.
+ *
+ * Useful for monitoring changes across multiple tables.
+ *
+ * @example
+ * ```typescript
+ * const merged = mergeCDCStreams([
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM users', key: 'id' }),
+ *   createLiveCDCStream(pg, { query: 'SELECT * FROM orders', key: 'id' }),
+ * ])
+ *
+ * for await (const change of merged) {
+ *   console.log('Change from some table:', change)
+ * }
+ * ```
+ */
+export declare function mergeCDCStreams<T = Record<string, unknown>>(streams: LiveCDCStream<T>[]): AsyncGenerator<CDCChange<T> & {
+    _sourceIndex: number;
+}, void, undefined>;
+//# sourceMappingURL=live-cdc-stream.d.ts.map

package/dist/streaming/live-cdc-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-cdc-stream.d.ts","sourceRoot":"","sources":["../../src/streaming/live-cdc-stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAMH;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAA;AAEvE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,qBAAqB;IACrB,MAAM,EAAE,gBAAgB,CAAA;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,EAAE,CAAA;IAC7B,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAE,SAAQ,aAAa;IAClF,MAAM,EAAE,QAAQ,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAE,SAAQ,aAAa;IAClF,MAAM,EAAE,QAAQ,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAE,SAAQ,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC;IACrG,MAAM,EAAE,QAAQ,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,aAAa;IACnD,MAAM,EAAE,OAAO,CAAA;IACf,mBAAmB,EAAE,EAAE,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAC7C,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GACxB,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GACxB,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GACxB,cAAc,CAAA;AAElB;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,EACrC,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,IAAI,EACzB,GAAG,CAAC,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,GAC3C,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAA;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC7D,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAA;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC5D,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACnD,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9B,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,KAAK,IAAI,CAAA;IAChE,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5E,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC9D,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,uBAAuB;IACvB,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,IAAI,CAAA;IACzB,wDAAwD;IACxD,GAAG,EAAE,MAAM,CAAA;IACX,mCAAmC;IACnC,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,wEAAwE;IACxE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gDAAgD;IAChD,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;IACzD,qDAAqD;IACrD,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,OAAO,CAAA;IAC1C,iEAAiE;IACjE,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAA;IACpB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAA;IAClB,2BAA2B;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,0BAA0B;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAA;IACxB,+BAA+B;IAC/B,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC1D,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAA;IACX,qBAAqB;IACrB,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACzC,kBAAkB;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,yBAAyB;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACpC,uCAAuC;IACvC,MAAM,CAAC,EAAE,CAAC,CAAA;IACV,uCAAuC;IACvC,MAAM,CAAC,EAAE,CAAC,CAAA;IACV,mCAAmC;IACnC,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,gBAAgB;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAA;CAChB;AAcD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,qBAAa,aAAa,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAE,YAAW,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5F,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAgB;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAC0C;IACjE,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,aAAa,CAAsD;IAC3E,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,UAAU,CAAI;gBAEV,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAqC9D;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAwCtC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB3B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,QAAQ,IAAI,cAAc;IAa1B;;OAEG;IACH,SAAS,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAOxD;;OAEG;IACH,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAuB9E;;OAEG;IACH,gBAAgB,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IA0ChD;;OAEG;IACH,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC;IAkDzC;;OAEG;IACH,cAAc,IAAI,cAAc,CAAC,UAAU,CAAC;IA6C5C;;;;OAIG;IACI,eAAe,CAAC,OAAO,GAAE;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,OAAO,CAAC,EAAE,MAAM,CAAA;KACZ,GAAG,cAAc,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAoD5D,OAAO,CAAC,aAAa;IAuBrB,OAAO,CAAC,cAAc;IA6CtB,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,KAAK;CASd;AAeD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7D,EAAE,EAAE,cAAc,EAClB,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC7B,aAAa,CAAC,CAAC,CAAC,CAElB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAuB,sBAAsB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvE,EAAE,EAAE,cAAc,EAClB,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAAG;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,GACA,cAAc,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAerD;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAuB,eAAe,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EACnC,OAAO,EAAE;IACP,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAA;CAClB,GACA,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CA4DjD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,eAAe,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,GAC1B,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,EAAE,IAAI,EAAE,SAAS,CAAC,CAkC1E"}