@powersync/service-module-postgres 0.13.0 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # @powersync/service-module-postgres
 
+## 0.14.0
+
+### Minor Changes
+
+- 0ccd470: Add powersync_replication_lag_seconds metric
+- 951b010: Implement resuming of initial replication snapshots.
+- d235f7b: [MongoDB Storage] Remove change streams on bucket storage database due to performance overhead.
+- 08b7aa9: Add checks for RLS affecting replication.
+
+### Patch Changes
+
+- 1907356: Cleanly interrupt clearing of storage when the process is stopped/restarted.
+- Updated dependencies [08b7aa9]
+- Updated dependencies [0ccd470]
+- Updated dependencies [1907356]
+- Updated dependencies [08b7aa9]
+- Updated dependencies [951b010]
+- Updated dependencies [d235f7b]
+- Updated dependencies [f9e8673]
+  - @powersync/service-core@1.13.0
+  - @powersync/service-types@0.12.0
+  - @powersync/service-jpgwire@0.20.0
+  - @powersync/lib-services-framework@0.7.0
+  - @powersync/lib-service-postgres@0.4.4
+
+## 0.13.1
+
+### Patch Changes
+
+- Updated dependencies [100ccec]
+- Updated dependencies [b57f938]
+- Updated dependencies [5b39039]
+  - @powersync/service-core@1.12.1
+  - @powersync/service-sync-rules@0.27.0
+
 ## 0.13.0
 
 ### Minor Changes

package/dist/api/PostgresRouteAPIAdapter.d.ts

@@ -20,7 +20,7 @@ export declare class PostgresRouteAPIAdapter implements api.RouteAPI {
     executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse>;
     getDebugTablesInfo(tablePatterns: sync_rules.TablePattern[], sqlSyncRules: sync_rules.SqlSyncRules): Promise<api.PatternResult[]>;
     protected getDebugTableInfo(tablePattern: sync_rules.TablePattern, name: string, relationId: number | null, syncRules: sync_rules.SqlSyncRules): Promise<service_types.TableInfo>;
-    getReplicationLag(options: api.ReplicationLagOptions): Promise<number | undefined>;
+    getReplicationLagBytes(options: api.ReplicationLagOptions): Promise<number | undefined>;
     getReplicationHead(): Promise<string>;
     createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
     getConnectionSchema(): Promise<service_types.DatabaseSchema[]>;

package/dist/api/PostgresRouteAPIAdapter.js

@@ -190,7 +190,7 @@ export class PostgresRouteAPIAdapter {
             syncRules: syncRules
         });
     }
-    async getReplicationLag(options) {
+    async getReplicationLagBytes(options) {
         const { bucketStorage } = options;
         const slotName = bucketStorage.slot_name;
         const results = await lib_postgres.retriedQuery(this.pool, {
@@ -224,6 +224,10 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
     async createReplicationHead(callback) {
         const currentLsn = await this.getReplicationHead();
         const r = await callback(currentLsn);
+        // Note: This may not reliably trigger a new replication message on Postgres 11 or 12,
+        // in which case there could be a delay in the client receiving the write checkpoint acknowledgement.
+        // Postgres 12 already reached EOL, and this is not a critical issue, so we're not fixing it.
+        // On postgres 13+, this works reliably.
         await lib_postgres.retriedQuery(this.pool, KEEPALIVE_STATEMENT);
         return r;
     }

package/dist/api/PostgresRouteAPIAdapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,uBAAuB;IAgBtB;IAEF;IAjBV,aAAa,CAAS;IACtB,oDAAoD;IACpD,eAAe,GAAG,gBAAgB,CAAC;IAEnC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,MAAM;SACpB,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAE/C,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC;aAC/C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;4BACpD,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;gBACxC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;8BAKS;oBACpB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;oBAC3F,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;2BAKM;oBACjB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC7B,kBAAkB;oBAClB,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC1G,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAC/B,YAAqC,EACrC,IAAY,EACZ,UAAyB,EACzB,SAAkC;QAElC,OAAO,iBAAiB,CAAC;YACvB,EAAE,EAAE,IAAI,CAAC,IAAI;YACb,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAkC;QACxD,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACzD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,YAAY,CAAC;YACrB,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,SAAS,CAAC,WAAW;YAC3B,WAAW,EAAE,gDAAgD,QAAQ,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAErG,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEnD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;QAErC,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAEhE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAuCsC,CACvC,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,SAAS;oBACzE,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}
+{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,uBAAuB;IAgBtB;IAEF;IAjBV,aAAa,CAAS;IACtB,oDAAoD;IACpD,eAAe,GAAG,gBAAgB,CAAC;IAEnC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,MAAM;SACpB,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAE/C,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC;aAC/C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;4BACpD,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;gBACxC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;8BAKS;oBACpB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;oBAC3F,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;2BAKM;oBACjB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC7B,kBAAkB;oBAClB,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC1G,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAC/B,YAAqC,EACrC,IAAY,EACZ,UAAyB,EACzB,SAAkC;QAElC,OAAO,iBAAiB,CAAC;YACvB,EAAE,EAAE,IAAI,CAAC,IAAI;YACb,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,OAAkC;QAC7D,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACzD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,YAAY,CAAC;YACrB,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,SAAS,CAAC,WAAW;YAC3B,WAAW,EAAE,gDAAgD,QAAQ,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAErG,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEnD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;QAErC,sFAAsF;QACtF,qGAAqG;QACrG,6FAA6F;QAC7F,wCAAwC;QACxC,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAEhE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAuCsC,CACvC,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,SAAS;oBACzE,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}

package/dist/replication/SnapshotQuery.d.ts

@@ -0,0 +1,78 @@
+import { SourceTable } from '@powersync/service-core';
+import { PgChunk, PgConnection, PgTypeOid } from '@powersync/service-jpgwire';
+import { SqliteValue } from '@powersync/service-sync-rules';
+export interface SnapshotQuery {
+    initialize(): Promise<void>;
+    /**
+     * Returns an async iterable iterator that yields chunks of data.
+     *
+     * If the last chunk has 0 rows, it indicates that there are no more rows to fetch.
+     */
+    nextChunk(): AsyncIterableIterator<PgChunk>;
+}
+export type PrimaryKeyValue = Record<string, SqliteValue>;
+export interface MissingRow {
+    table: SourceTable;
+    key: PrimaryKeyValue;
+}
+/**
+ * Snapshot query using a plain SELECT * FROM table; chunked using
+ * DELCLARE CURSOR / FETCH.
+ *
+ * This supports all tables, but does not efficiently resume the snapshot
+ * if the process is restarted.
+ */
+export declare class SimpleSnapshotQuery implements SnapshotQuery {
+    private readonly connection;
+    private readonly table;
+    private readonly chunkSize;
+    constructor(connection: PgConnection, table: SourceTable, chunkSize?: number);
+    initialize(): Promise<void>;
+    nextChunk(): AsyncIterableIterator<PgChunk>;
+}
+/**
+ * Performs a table snapshot query, chunking by ranges of primary key data.
+ *
+ * This may miss some rows if they are modified during the snapshot query.
+ * In that case, logical replication will pick up those rows afterwards,
+ * possibly resulting in an IdSnapshotQuery.
+ *
+ * Currently, this only supports a table with a single primary key column,
+ * of a select few types.
+ */
+export declare class ChunkedSnapshotQuery implements SnapshotQuery {
+    private readonly connection;
+    private readonly table;
+    private readonly chunkSize;
+    /**
+     * Primary key types that we support for chunked snapshots.
+     *
+     * Can expand this over time as we add more tests,
+     * and ensure there are no issues with type conversion.
+     */
+    static SUPPORTED_TYPES: PgTypeOid[];
+    static supports(table: SourceTable): boolean;
+    private readonly key;
+    lastKey: string | bigint | null;
+    constructor(connection: PgConnection, table: SourceTable, chunkSize: number | undefined, lastKeySerialized: Uint8Array | null);
+    initialize(): Promise<void>;
+    private deserializeKey;
+    getLastKeySerialized(): Uint8Array;
+    nextChunk(): AsyncIterableIterator<PgChunk>;
+}
+/**
+ * This performs a snapshot query using a list of primary keys.
+ *
+ * This is not used for general snapshots, but is used when we need to re-fetch specific rows
+ * during streaming replication.
+ */
+export declare class IdSnapshotQuery implements SnapshotQuery {
+    private readonly connection;
+    private readonly table;
+    private readonly keys;
+    private didChunk;
+    static supports(table: SourceTable): boolean;
+    constructor(connection: PgConnection, table: SourceTable, keys: PrimaryKeyValue[]);
+    initialize(): Promise<void>;
+    nextChunk(): AsyncIterableIterator<PgChunk>;
+}

package/dist/replication/SnapshotQuery.js

@@ -0,0 +1,175 @@
+import { bson } from '@powersync/service-core';
+import { PgType, PgTypeOid } from '@powersync/service-jpgwire';
+import { escapeIdentifier } from '@powersync/lib-service-postgres';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+/**
+ * Snapshot query using a plain SELECT * FROM table; chunked using
+ * DELCLARE CURSOR / FETCH.
+ *
+ * This supports all tables, but does not efficiently resume the snapshot
+ * if the process is restarted.
+ */
+export class SimpleSnapshotQuery {
+    connection;
+    table;
+    chunkSize;
+    constructor(connection, table, chunkSize = 10_000) {
+        this.connection = connection;
+        this.table = table;
+        this.chunkSize = chunkSize;
+    }
+    async initialize() {
+        await this.connection.query(`DECLARE snapshot_cursor CURSOR FOR SELECT * FROM ${this.table.escapedIdentifier}`);
+    }
+    nextChunk() {
+        return this.connection.stream(`FETCH ${this.chunkSize} FROM snapshot_cursor`);
+    }
+}
+/**
+ * Performs a table snapshot query, chunking by ranges of primary key data.
+ *
+ * This may miss some rows if they are modified during the snapshot query.
+ * In that case, logical replication will pick up those rows afterwards,
+ * possibly resulting in an IdSnapshotQuery.
+ *
+ * Currently, this only supports a table with a single primary key column,
+ * of a select few types.
+ */
+export class ChunkedSnapshotQuery {
+    connection;
+    table;
+    chunkSize;
+    /**
+     * Primary key types that we support for chunked snapshots.
+     *
+     * Can expand this over time as we add more tests,
+     * and ensure there are no issues with type conversion.
+     */
+    static SUPPORTED_TYPES = [
+        PgTypeOid.TEXT,
+        PgTypeOid.VARCHAR,
+        PgTypeOid.UUID,
+        PgTypeOid.INT2,
+        PgTypeOid.INT4,
+        PgTypeOid.INT8
+    ];
+    static supports(table) {
+        if (table.replicaIdColumns.length != 1) {
+            return false;
+        }
+        const primaryKey = table.replicaIdColumns[0];
+        return primaryKey.typeId != null && ChunkedSnapshotQuery.SUPPORTED_TYPES.includes(Number(primaryKey.typeId));
+    }
+    key;
+    lastKey = null;
+    constructor(connection, table, chunkSize = 10_000, lastKeySerialized) {
+        this.connection = connection;
+        this.table = table;
+        this.chunkSize = chunkSize;
+        this.key = table.replicaIdColumns[0];
+        if (lastKeySerialized != null) {
+            this.lastKey = this.deserializeKey(lastKeySerialized);
+        }
+    }
+    async initialize() {
+        // No-op
+    }
+    deserializeKey(key) {
+        const decoded = bson.deserialize(key, { useBigInt64: true });
+        const keys = Object.keys(decoded);
+        if (keys.length != 1) {
+            throw new ServiceAssertionError(`Multiple keys found: ${keys.join(', ')}`);
+        }
+        if (keys[0] != this.key.name) {
+            throw new ServiceAssertionError(`Key name mismatch: expected ${this.key.name}, got ${keys[0]}`);
+        }
+        const value = decoded[this.key.name];
+        return value;
+    }
+    getLastKeySerialized() {
+        return bson.serialize({ [this.key.name]: this.lastKey });
+    }
+    async *nextChunk() {
+        let stream;
+        const escapedKeyName = escapeIdentifier(this.key.name);
+        if (this.lastKey == null) {
+            stream = this.connection.stream(`SELECT * FROM ${this.table.escapedIdentifier} ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}`);
+        }
+        else {
+            if (this.key.typeId == null) {
+                throw new Error(`typeId required for primary key ${this.key.name}`);
+            }
+            let type = Number(this.key.typeId);
+            stream = this.connection.stream({
+                statement: `SELECT * FROM ${this.table.escapedIdentifier} WHERE ${escapedKeyName} > $1 ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}`,
+                params: [{ value: this.lastKey, type }]
+            });
+        }
+        let primaryKeyIndex = -1;
+        for await (let chunk of stream) {
+            if (chunk.tag == 'RowDescription') {
+                // We get a RowDescription for each FETCH call, but they should
+                // all be the same.
+                let i = 0;
+                const pk = chunk.payload.findIndex((c) => c.name == this.key.name);
+                if (pk < 0) {
+                    throw new Error(`Cannot find primary key column ${this.key} in results. Keys: ${chunk.payload.map((c) => c.name).join(', ')}`);
+                }
+                primaryKeyIndex = pk;
+            }
+            if (chunk.rows.length > 0) {
+                this.lastKey = chunk.rows[chunk.rows.length - 1][primaryKeyIndex];
+            }
+            yield chunk;
+        }
+    }
+}
+/**
+ * This performs a snapshot query using a list of primary keys.
+ *
+ * This is not used for general snapshots, but is used when we need to re-fetch specific rows
+ * during streaming replication.
+ */
+export class IdSnapshotQuery {
+    connection;
+    table;
+    keys;
+    didChunk = false;
+    static supports(table) {
+        // We have the same requirements as ChunkedSnapshotQuery.
+        // This is typically only used as a fallback when ChunkedSnapshotQuery
+        // skipped some rows.
+        return ChunkedSnapshotQuery.supports(table);
+    }
+    constructor(connection, table, keys) {
+        this.connection = connection;
+        this.table = table;
+        this.keys = keys;
+    }
+    async initialize() {
+        // No-op
+    }
+    async *nextChunk() {
+        // Only produce one chunk
+        if (this.didChunk) {
+            return;
+        }
+        this.didChunk = true;
+        const keyDefinition = this.table.replicaIdColumns[0];
+        const ids = this.keys.map((record) => record[keyDefinition.name]);
+        const type = PgType.getArrayType(keyDefinition.typeId);
+        if (type == null) {
+            throw new Error(`Cannot determine primary key array type for ${JSON.stringify(keyDefinition)}`);
+        }
+        yield* this.connection.stream({
+            statement: `SELECT * FROM ${this.table.escapedIdentifier} WHERE ${escapeIdentifier(keyDefinition.name)} = ANY($1)`,
+            params: [
+                {
+                    type: type,
+                    value: ids
+                }
+            ]
+        });
+    }
+}
+//# sourceMappingURL=SnapshotQuery.js.map

package/dist/replication/SnapshotQuery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SnapshotQuery.js","sourceRoot":"","sources":["../../src/replication/SnapshotQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiC,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAyB,MAAM,EAAkB,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACtG,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAEnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAmB1E;;;;;;GAMG;AACH,MAAM,OAAO,mBAAmB;IAEX;IACA;IACA;IAHnB,YACmB,UAAwB,EACxB,KAAkB,EAClB,YAAoB,MAAM;QAF1B,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAiB;IAC1C,CAAC;IAEG,KAAK,CAAC,UAAU;QACrB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,oDAAoD,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAClH,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,SAAS,uBAAuB,CAAC,CAAC;IAChF,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAoB;IA6BZ;IACA;IACA;IA9BnB;;;;;OAKG;IACH,MAAM,CAAC,eAAe,GAAG;QACvB,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,OAAO;QACjB,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;KACf,CAAC;IAEF,MAAM,CAAC,QAAQ,CAAC,KAAkB;QAChC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE7C,OAAO,UAAU,CAAC,MAAM,IAAI,IAAI,IAAI,oBAAoB,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/G,CAAC;IAEgB,GAAG,CAAmB;IACvC,OAAO,GAA2B,IAAI,CAAC;IAEvC,YACmB,UAAwB,EACxB,KAAkB,EAClB,YAAoB,MAAM,EAC3C,iBAAoC;QAHnB,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAiB;QAG3C,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAErC,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,UAAU;QACrB,QAAQ;IACV,CAAC;IAEO,cAAc,CAAC,GAAe;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,oBAAoB;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;IAEM,KAAK,CAAC,CAAC,SAAS;QACrB,IAAI,MAAsC,CAAC;QAC3C,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAC7B,iBAAiB,IAAI,CAAC,KAAK,CAAC,iBAAiB,aAAa,cAAc,UAAU,IAAI,CAAC,SAAS,EAAE,CACnG,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACtE,CAAC;YACD,IAAI,IAAI,GAA2B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC9B,SAAS,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,iBAAiB,UAAU,cAAc,kBAAkB,cAAc,UAAU,IAAI,CAAC,SAAS,EAAE;gBAC1I,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,eAAe,GAAW,CAAC,CAAC,CAAC;QAEjC,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,GAAG,IAAI,gBAAgB,EAAE,CAAC;gBAClC,+DAA+D;gBAC/D,mBAAmB;gBACnB,IAAI,CAAC,GAAG,CAAC,CAAC;gBACV,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnE,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBACX,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,GAAG,sBAAsB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9G,CAAC;gBACJ,CAAC;gBACD,eAAe,GAAG,EAAE,CAAC;YACvB,CAAC;YAED,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YACpE,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;;AAGH;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IAWP;IACA;IACA;IAZX,QAAQ,GAAG,KAAK,CAAC;IAEzB,MAAM,CAAC,QAAQ,CAAC,KAAkB;QAChC,yDAAyD;QACzD,sEAAsE;QACtE,qBAAqB;QACrB,OAAO,oBAAoB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,YACmB,UAAwB,EACxB,KAAkB,EAClB,IAAuB;QAFvB,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,SAAI,GAAJ,IAAI,CAAmB;IACvC,CAAC;IAEG,KAAK,CAAC,UAAU;QACrB,QAAQ;IACV,CAAC;IAEM,KAAK,CAAC,CAAC,SAAS;QACrB,yBAAyB;QACzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,MAAO,CAAC,CAAC;QACxD,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC5B,SAAS,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,iBAAiB,UAAU,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY;YAClH,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,GAAG;iBACX;aACF;SACF,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/replication/WalStream.d.ts

@@ -1,12 +1,22 @@
-import { MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core';
+import { Logger } from '@powersync/lib-services-framework';
+import { MetricsEngine, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
 import { PgManager } from './PgManager.js';
 export interface WalStreamOptions {
+    logger?: Logger;
     connections: PgManager;
     storage: storage.SyncRulesBucketStorage;
     metrics: MetricsEngine;
     abort_signal: AbortSignal;
+    /**
+     * Override snapshot chunk length (number of rows), for testing.
+     *
+     * Defaults to 10_000.
+     *
+     * Note that queries are streamed, so we don't actually keep that much data in memory.
+     */
+    snapshotChunkLength?: number;
 }
 interface InitResult {
     /** True if initial snapshot is not yet done. */
@@ -29,13 +39,25 @@ export declare class WalStream {
     sync_rules: SqlSyncRules;
     group_id: number;
     connection_id: number;
+    private logger;
     private readonly storage;
     private readonly metrics;
     private readonly slot_name;
     private connections;
     private abort_signal;
-    private relation_cache;
+    private relationCache;
     private startedStreaming;
+    private snapshotChunkLength;
+    /**
+     * Time of the oldest uncommitted change, according to the source db.
+     * This is used to determine the replication lag.
+     */
+    private oldestUncommittedChange;
+    /**
+     * Keep track of whether we have done a commit or keepalive yet.
+     * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
+     */
+    private isStartingReplication;
     constructor(options: WalStreamOptions);
     get stopped(): boolean;
     getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
@@ -44,7 +66,7 @@ export declare class WalStream {
      * If a replication slot exists, check that it is healthy.
      */
     private checkReplicationSlot;
-    estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string>;
+    estimatedCountNumber(db: pgwire.PgConnection, table: storage.SourceTable): Promise<number>;
     /**
      * Start initial replication.
      *
@@ -54,8 +76,18 @@ export declare class WalStream {
     startInitialReplication(replicationConnection: pgwire.PgConnection, status: InitResult): Promise<void>;
     initialReplication(db: pgwire.PgConnection): Promise<void>;
     static getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow>;
+    private snapshotTableInTx;
     private snapshotTable;
-    handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean): Promise<storage.SourceTable>;
+    handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean): Promise<SourceTable>;
+    /**
+     * Process rows that have missing TOAST values.
+     *
+     * This can happen during edge cases in the chunked intial snapshot process.
+     *
+     * We handle this similar to an inline table snapshot, but limited to the specific
+     * set of rows.
+     */
+    private resnapshot;
     private getTable;
     writeChange(batch: storage.BucketStorageBatch, msg: pgwire.PgoutputMessage): Promise<storage.FlushedResult | null>;
     replicate(): Promise<void>;
@@ -72,5 +104,6 @@ export declare class WalStream {
      * viewing the contents of logical replication messages.
      */
     protected checkLogicalMessageSupport(): Promise<boolean>;
+    getReplicationLagMillis(): Promise<number | undefined>;
 }
 export {};