@powersync/service-module-postgres 0.16.9 → 0.16.11

package/dist/replication/WalStreamReplicationJob.js

@@ -1,21 +1,15 @@
-import { container, logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { container, logger } from '@powersync/lib-services-framework';
 import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
 import { replication } from '@powersync/service-core';
 import { getApplicationName } from '../utils/application-name.js';
 export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
     connectionFactory;
-    connectionManager;
+    connectionManager = null;
     lastStream = null;
     constructor(options) {
         super(options);
         this.logger = logger.child({ prefix: `[${this.slotName}] ` });
         this.connectionFactory = options.connectionFactory;
-        this.connectionManager = this.connectionFactory.create({
-            // Pool connections are only used intermittently.
-            idleTimeout: 30_000,
-            maxSize: 2,
-            applicationName: getApplicationName()
-        });
     }
     /**
      * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
@@ -30,11 +24,13 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
      * **This may be a bug in pgwire or how we're using it.
      */
     async keepAlive() {
-        try {
-            await sendKeepAlive(this.connectionManager.pool);
-        }
-        catch (e) {
-            this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+        if (this.connectionManager) {
+            try {
+                await sendKeepAlive(this.connectionManager.pool);
+            }
+            catch (e) {
+                this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+            }
         }
     }
     get slotName() {
@@ -42,33 +38,55 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
     }
     async replicate() {
         try {
-            await this.replicateLoop();
+            await this.replicateOnce();
         }
         catch (e) {
             // Fatal exception
-            container.reporter.captureException(e, {
-                metadata: {
-                    replication_slot: this.slotName
+            if (!this.isStopped) {
+                // Ignore aborted errors
+                this.logger.error(`Replication error`, e);
+                if (e.cause != null) {
+                    // Example:
+                    // PgError.conn_ended: Unable to do postgres query on ended connection
+                    //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
+                    //     at stream.next (<anonymous>)
+                    //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
+                    //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
+                    //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
+                    //     ...
+                    //   cause: TypeError: match is not iterable
+                    //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
+                    //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
+                    //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
+                    //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
+                    //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
+                    //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
+                    //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
+                    //   [Symbol(pg.ErrorCode)]: 'conn_ended',
+                    //   [Symbol(pg.ErrorResponse)]: undefined
+                    // }
+                    // Without this additional log, the cause would not be visible in the logs.
+                    this.logger.error(`cause`, e.cause);
                 }
-            });
-            this.logger.error(`Replication failed`, e);
+                // Report the error if relevant, before retrying
+                container.reporter.captureException(e, {
+                    metadata: {
+                        replication_slot: this.slotName
+                    }
+                });
+                // This sets the retry delay
+                this.rateLimiter.reportError(e);
+            }
             if (e instanceof MissingReplicationSlotError) {
                 // This stops replication on this slot and restarts with a new slot
                 await this.options.storage.factory.restartReplication(this.storage.group_id);
             }
+            // No need to rethrow - the error is already logged, and retry behavior is the same on error
         }
         finally {
             this.abortController.abort();
         }
     }
-    async replicateLoop() {
-        while (!this.isStopped) {
-            await this.replicateOnce();
-            if (!this.isStopped) {
-                await new Promise((resolve) => setTimeout(resolve, 5000));
-            }
-        }
-    }
     async replicateOnce() {
         // New connections on every iteration (every error with retry),
         // otherwise we risk repeating errors related to the connection,
@@ -79,6 +97,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
             maxSize: 2,
             applicationName: getApplicationName()
         });
+        this.connectionManager = connectionManager;
         try {
             await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
             if (this.isStopped) {
@@ -94,50 +113,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
             this.lastStream = stream;
             await stream.replicate();
         }
-        catch (e) {
-            if (this.isStopped && e instanceof ReplicationAbortedError) {
-                // Ignore aborted errors
-                return;
-            }
-            this.logger.error(`Replication error`, e);
-            if (e.cause != null) {
-                // Example:
-                // PgError.conn_ended: Unable to do postgres query on ended connection
-                //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-                //     at stream.next (<anonymous>)
-                //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-                //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-                //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-                //     ...
-                //   cause: TypeError: match is not iterable
-                //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-                //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-                //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-                //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-                //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-                //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-                //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-                //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-                //   [Symbol(pg.ErrorResponse)]: undefined
-                // }
-                // Without this additional log, the cause would not be visible in the logs.
-                this.logger.error(`cause`, e.cause);
-            }
-            if (e instanceof MissingReplicationSlotError) {
-                throw e;
-            }
-            else {
-                // Report the error if relevant, before retrying
-                container.reporter.captureException(e, {
-                    metadata: {
-                        replication_slot: this.slotName
-                    }
-                });
-                // This sets the retry delay
-                this.rateLimiter?.reportError(e);
-            }
-        }
         finally {
+            this.connectionManager = null;
             await connectionManager.end();
         }
     }

package/dist/replication/WalStreamReplicationJob.js.map

@@ -1 +1 @@
-{"version":3,"file":"WalStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/WalStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAE/F,OAAO,EAAE,2BAA2B,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEvF,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAMlE,MAAM,OAAO,uBAAwB,SAAQ,WAAW,CAAC,sBAAsB;IACrE,iBAAiB,CAA2B;IACnC,iBAAiB,CAAY;IACtC,UAAU,GAAqB,IAAI,CAAC;IAE5C,YAAY,OAAuC;QACjD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACrD,iDAAiD;YACjD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;gBACrC,QAAQ,EAAE;oBACR,gBAAgB,EAAE,IAAI,CAAC,QAAQ;iBAChC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YAE3C,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,mEAAmE;gBACnE,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iDAAiD;YACjD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,uBAAuB,EAAE,CAAC;gBAC3D,wBAAwB;gBACxB,OAAO;YACT,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpB,WAAW;gBACX,sEAAsE;gBACtE,2KAA2K;gBAC3K,mCAAmC;gBACnC,4KAA4K;gBAC5K,0KAA0K;gBAC1K,yHAAyH;gBACzH,UAAU;gBACV,4CAA4C;gBAC5C,4FAA4F;gBAC5F,6FAA6F;gBAC7F,iGAAiG;gBACjG,oLAAoL;gBACpL,qLAAqL;gBACrL,sFAAsF;gBACtF,qLAAqL;gBACrL,0CAA0C;gBAC1C,0CAA0C;gBAC1C,IAAI;gBACJ,2EAA2E;gBAC3E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;YACD,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,MAAM,CAAC,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,gDAAgD;gBAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE;wBACR,gBAAgB,EAAE,IAAI,CAAC,QAAQ;qBAChC;iBACF,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
+{"version":3,"file":"WalStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/WalStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAA2B,MAAM,mCAAmC,CAAC;AAE/F,OAAO,EAAE,2BAA2B,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEvF,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAMlE,MAAM,OAAO,uBAAwB,SAAQ,WAAW,CAAC,sBAAsB;IACrE,iBAAiB,CAA2B;IAC5C,iBAAiB,GAAqB,IAAI,CAAC;IAC3C,UAAU,GAAqB,IAAI,CAAC;IAE5C,YAAY,OAAuC;QACjD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAElB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBAExB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,WAAW;oBACX,sEAAsE;oBACtE,2KAA2K;oBAC3K,mCAAmC;oBACnC,4KAA4K;oBAC5K,0KAA0K;oBAC1K,yHAAyH;oBACzH,UAAU;oBACV,4CAA4C;oBAC5C,4FAA4F;oBAC5F,6FAA6F;oBAC7F,iGAAiG;oBACjG,oLAAoL;oBACpL,qLAAqL;oBACrL,sFAAsF;oBACtF,qLAAqL;oBACrL,0CAA0C;oBAC1C,0CAA0C;oBAC1C,IAAI;oBACJ,2EAA2E;oBAC3E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBACD,gDAAgD;gBAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE;wBACR,gBAAgB,EAAE,IAAI,CAAC,QAAQ;qBAChC;iBACF,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,mEAAmE;gBACnE,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;YAED,4FAA4F;QAC9F,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iDAAiD;YACjD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.16.9",
+  "version": "0.16.11",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -30,7 +30,7 @@
     "uuid": "^11.1.0",
     "@powersync/lib-service-postgres": "0.4.13",
     "@powersync/lib-services-framework": "0.7.9",
-    "@powersync/service-core": "1.16.0",
+    "@powersync/service-core": "1.16.2",
     "@powersync/service-jpgwire": "0.21.5",
     "@powersync/service-jsonbig": "0.17.12",
     "@powersync/service-sync-rules": "0.29.6",
@@ -38,10 +38,10 @@
   },
   "devDependencies": {
     "@types/semver": "^7.5.4",
-    "@powersync/service-core-tests": "0.12.9",
-    "@powersync/service-module-mongodb-storage": "0.12.9",
+    "@powersync/service-core-tests": "0.12.11",
+    "@powersync/service-module-mongodb-storage": "0.12.11",
     "@powersync/lib-service-postgres": "0.4.13",
-    "@powersync/service-module-postgres-storage": "0.10.9"
+    "@powersync/service-module-postgres-storage": "0.10.11"
   },
   "scripts": {
     "build": "tsc -b",

package/src/replication/ConnectionManagerFactory.ts

@@ -5,7 +5,7 @@ import { logger } from '@powersync/lib-services-framework';
 import { CustomTypeRegistry } from '../types/registry.js';
 
 export class ConnectionManagerFactory {
-  private readonly connectionManagers: PgManager[];
+  private readonly connectionManagers = new Set<PgManager>();
   public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
 
   constructor(
@@ -13,18 +13,23 @@ export class ConnectionManagerFactory {
     private readonly registry: CustomTypeRegistry
   ) {
     this.dbConnectionConfig = dbConnectionConfig;
-    this.connectionManagers = [];
   }
 
   create(poolOptions: PgPoolOptions) {
     const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions, registry: this.registry });
-    this.connectionManagers.push(manager);
+    this.connectionManagers.add(manager);
+
+    manager.registerListener({
+      onEnded: () => {
+        this.connectionManagers.delete(manager);
+      }
+    });
     return manager;
   }
 
   async shutdown() {
     logger.info('Shutting down Postgres connection Managers...');
-    for (const manager of this.connectionManagers) {
+    for (const manager of [...this.connectionManagers]) {
       await manager.end();
     }
     logger.info('Postgres connection Managers shutdown completed.');

package/src/replication/PgManager.ts

@@ -5,6 +5,7 @@ import { getApplicationName } from '../utils/application-name.js';
 import { PostgresTypeResolver } from '../types/resolver.js';
 import { getServerVersion } from '../utils/postgres_version.js';
 import { CustomTypeRegistry } from '../types/registry.js';
+import { BaseObserver } from '@powersync/lib-services-framework';
 
 export interface PgManagerOptions extends pgwire.PgPoolOptions {
   registry: CustomTypeRegistry;
@@ -15,7 +16,11 @@ export interface PgManagerOptions extends pgwire.PgPoolOptions {
  */
 const SNAPSHOT_SOCKET_TIMEOUT = 30_000;
 
-export class PgManager {
+export interface PgManagerListener {
+  onEnded(): void;
+}
+
+export class PgManager extends BaseObserver<PgManagerListener> {
   /**
    * Do not use this for any transactions.
    */
@@ -29,6 +34,7 @@ export class PgManager {
     public options: NormalizedPostgresConnectionConfig,
     public poolOptions: PgManagerOptions
   ) {
+    super();
     // The pool is lazy - no connections are opened until a query is performed.
     this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
     this.types = new PostgresTypeResolver(poolOptions.registry, this.pool);
@@ -83,8 +89,9 @@ export class PgManager {
     for (let result of await Promise.allSettled([
       this.pool.end(),
       ...this.connectionPromises.map(async (promise) => {
-        const connection = await promise;
-        return await connection.end();
+        // Wait for connection attempts to finish, but do not throw connection errors here
+        const connection = await promise.catch((_) => {});
+        return await connection?.end();
       })
     ])) {
       // Throw the first error, if any
@@ -92,14 +99,18 @@ export class PgManager {
         throw result.reason;
       }
     }
+    this.iterateListeners((listener) => {
+      listener.onEnded?.();
+    });
   }
 
   async destroy() {
     this.pool.destroy();
     for (let result of await Promise.allSettled([
       ...this.connectionPromises.map(async (promise) => {
-        const connection = await promise;
-        return connection.destroy();
+        // Wait for connection attempts to finish, but do not throw connection errors here
+        const connection = await promise.catch((_) => {});
+        return connection?.destroy();
       })
     ])) {
       // Throw the first error, if any
@@ -107,5 +118,8 @@ export class PgManager {
         throw result.reason;
       }
     }
+    this.iterateListeners((listener) => {
+      listener.onEnded?.();
+    });
   }
 }

package/src/replication/PostgresErrorRateLimiter.ts

@@ -1,5 +1,6 @@
 import { setTimeout } from 'timers/promises';
 import { ErrorRateLimiter } from '@powersync/service-core';
+import { MissingReplicationSlotError } from './WalStream.js';
 
 export class PostgresErrorRateLimiter implements ErrorRateLimiter {
   nextAllowed: number = Date.now();
@@ -17,7 +18,10 @@ export class PostgresErrorRateLimiter implements ErrorRateLimiter {
 
   reportError(e: any): void {
     const message = (e.message as string) ?? '';
-    if (message.includes('password authentication failed')) {
+    if (e instanceof MissingReplicationSlotError) {
+      // Short delay for a retrying (re-creating the slot)
+      this.setDelay(2_000);
+    } else if (message.includes('password authentication failed')) {
       // Wait 15 minutes, to avoid triggering Supabase's fail2ban
       this.setDelay(900_000);
     } else if (message.includes('ENOTFOUND')) {

package/src/replication/WalStream.ts

@@ -4,7 +4,6 @@ import {
   DatabaseConnectionError,
   logger as defaultLogger,
   ErrorCode,
-  errors,
   Logger,
   ReplicationAbortedError,
   ReplicationAssertionError
@@ -100,8 +99,10 @@ export const sendKeepAlive = async (db: pgwire.PgClient) => {
 };
 
 export class MissingReplicationSlotError extends Error {
-  constructor(message: string) {
+  constructor(message: string, cause?: any) {
     super(message);
+
+    this.cause = cause;
   }
 }
 
@@ -304,135 +305,54 @@ export class WalStream {
       })
     )[0];
 
+    // Previously we also used pg_catalog.pg_logical_slot_peek_binary_changes to confirm that we can query the slot.
+    // However, there were some edge cases where the query times out, repeating the query, ultimately
+    // causing high load on the source database and never recovering automatically.
+    // We now instead jump straight to replication if the wal_status is not "lost", rather detecting those
+    // errors during streaming replication, which is a little more robust.
+
+    // We can have:
+    //   1. needsInitialSync: true, lost slot -> MissingReplicationSlotError (starts new sync rules version).
+    //      Theoretically we could handle this the same as (2).
+    //   2. needsInitialSync: true, no slot -> create new slot
+    //   3. needsInitialSync: true, valid slot -> resume initial sync
+    //   4. needsInitialSync: false, lost slot -> MissingReplicationSlotError (starts new sync rules version)
+    //   5. needsInitialSync: false, no slot -> MissingReplicationSlotError (starts new sync rules version)
+    //   6. needsInitialSync: false, valid slot -> resume streaming replication
+    // The main advantage of MissingReplicationSlotError are:
+    // 1. If there was a complete snapshot already (cases 4/5), users can still sync from that snapshot while
+    //    we do the reprocessing under a new slot name.
+    // 2. If there was a partial snapshot (case 1), we can start with the new slot faster by not waiting for
+    //    the partial data to be cleared.
     if (slot != null) {
       // This checks that the slot is still valid
-      const r = await this.checkReplicationSlot(slot as any);
-      if (snapshotDone && r.needsNewSlot) {
-        // We keep the current snapshot, and create a new replication slot
-        throw new MissingReplicationSlotError(`Replication slot ${slotName} is not valid anymore`);
+
+      // wal_status is present in postgres 13+
+      // invalidation_reason is present in postgres 17+
+      const lost = slot.wal_status == 'lost';
+      if (lost) {
+        // Case 1 / 4
+        throw new MissingReplicationSlotError(
+          `Replication slot ${slotName} is not valid anymore. invalidation_reason: ${slot.invalidation_reason ?? 'unknown'}`
+        );
       }
-      // We can have:
-      //   needsInitialSync: true, needsNewSlot: true -> initial sync from scratch
-      //   needsInitialSync: true, needsNewSlot: false -> resume initial sync
-      //   needsInitialSync: false, needsNewSlot: true -> handled above
-      //   needsInitialSync: false, needsNewSlot: false -> resume streaming replication
+      // Case 3 / 6
       return {
         needsInitialSync: !snapshotDone,
-        needsNewSlot: r.needsNewSlot
+        needsNewSlot: false
       };
     } else {
       if (snapshotDone) {
+        // Case 5
         // This will create a new slot, while keeping the current sync rules active
         throw new MissingReplicationSlotError(`Replication slot ${slotName} is missing`);
       }
-      // This will clear data and re-create the same slot
+      // Case 2
+      // This will clear data (if any) and re-create the same slot
       return { needsInitialSync: true, needsNewSlot: true };
     }
   }
 
-  /**
-   * If a replication slot exists, check that it is healthy.
-   */
-  private async checkReplicationSlot(slot: {
-    // postgres 13+
-    wal_status?: string;
-    // postgres 17+
-    invalidation_reason?: string | null;
-  }): Promise<{ needsNewSlot: boolean }> {
-    // Start with a placeholder error, should be replaced if there is an actual issue.
-    let last_error = new ReplicationAssertionError(`Slot health check failed to execute`);
-
-    const slotName = this.slot_name;
-
-    const lost = slot.wal_status == 'lost';
-    if (lost) {
-      this.logger.warn(
-        `Replication slot ${slotName} is invalidated. invalidation_reason: ${slot.invalidation_reason ?? 'unknown'}`
-      );
-      return {
-        needsNewSlot: true
-      };
-    }
-
-    // Check that replication slot exists, trying for up to 2 minutes.
-    const startAt = performance.now();
-    while (performance.now() - startAt < 120_000) {
-      this.touch();
-
-      try {
-        // We peek a large number of changes here, to make it more likely to pick up replication slot errors.
-        // For example, "publication does not exist" only occurs here if the peek actually includes changes related
-        // to the slot.
-        this.logger.info(`Checking ${slotName}`);
-
-        // The actual results can be quite large, so we don't actually return everything
-        // due to memory and processing overhead that would create.
-        const cursor = await this.connections.pool.stream({
-          statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
-          params: [
-            { type: 'varchar', value: slotName },
-            { type: 'varchar', value: PUBLICATION_NAME }
-          ]
-        });
-
-        for await (let _chunk of cursor) {
-          // No-op, just exhaust the cursor
-        }
-
-        // Success
-        this.logger.info(`Slot ${slotName} appears healthy`);
-        return { needsNewSlot: false };
-      } catch (e) {
-        last_error = e;
-        this.logger.warn(`Replication slot error`, e);
-
-        if (this.stopped) {
-          throw e;
-        }
-
-        if (
-          /incorrect prev-link/.test(e.message) ||
-          /replication slot.*does not exist/.test(e.message) ||
-          /publication.*does not exist/.test(e.message) ||
-          // Postgres 18 - exceeded max_slot_wal_keep_size
-          /can no longer access replication slot/.test(e.message) ||
-          // Postgres 17 - exceeded max_slot_wal_keep_size
-          /can no longer get changes from replication slot/.test(e.message)
-        ) {
-          // Fatal error. In most cases since Postgres 13+, the `wal_status == 'lost'` check should pick this up, but this
-          // works as a fallback.
-
-          container.reporter.captureException(e, {
-            level: errors.ErrorSeverity.WARNING,
-            metadata: {
-              replication_slot: slotName
-            }
-          });
-          // Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
-          //   Seen during development. Some internal error, fixed by re-creating slot.
-          //
-          // Sample: publication "powersync" does not exist
-          //   Happens when publication deleted or never created.
-          //   Slot must be re-created in this case.
-          this.logger.info(`${slotName} is not valid anymore`);
-
-          return { needsNewSlot: true };
-        }
-        // Try again after a pause
-        await new Promise((resolve) => setTimeout(resolve, 1000));
-      }
-    }
-
-    container.reporter.captureException(last_error, {
-      level: errors.ErrorSeverity.ERROR,
-      metadata: {
-        replication_slot: slotName
-      }
-    });
-
-    throw last_error;
-  }
-
   async estimatedCountNumber(db: pgwire.PgConnection, table: storage.SourceTable): Promise<number> {
     const results = await db.query({
       statement: `SELECT reltuples::bigint AS estimate
@@ -915,6 +835,17 @@ WHERE  oid = $1::regclass`,
   }
 
   async streamChanges(replicationConnection: pgwire.PgConnection) {
+    try {
+      await this.streamChangesInternal(replicationConnection);
+    } catch (e) {
+      if (isReplicationSlotInvalidError(e)) {
+        throw new MissingReplicationSlotError(e.message, e);
+      }
+      throw e;
+    }
+  }
+
+  private async streamChangesInternal(replicationConnection: pgwire.PgConnection) {
     // When changing any logic here, check /docs/wal-lsns.md.
     const { createEmptyCheckpoints } = await this.ensureStorageCompatibility();
 
@@ -1179,3 +1110,27 @@ WHERE  oid = $1::regclass`,
     });
   }
 }
+
+function isReplicationSlotInvalidError(e: any) {
+  // We could access the error code from pgwire using this:
+  //   e[Symbol.for('pg.ErrorCode')]
+  // However, we typically get a generic code such as 42704 (undefined_object), which does not
+  // help much. So we check the actual error message.
+  const message = e.message ?? '';
+
+  // Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
+  //   Seen during development. Some internal error, fixed by re-creating slot.
+  //
+  // Sample: publication "powersync" does not exist
+  //   Happens when publication deleted or never created.
+  //   Slot must be re-created in this case.
+  return (
+    /incorrect prev-link/.test(message) ||
+    /replication slot.*does not exist/.test(message) ||
+    /publication.*does not exist/.test(message) ||
+    // Postgres 18 - exceeded max_slot_wal_keep_size
+    /can no longer access replication slot/.test(message) ||
+    // Postgres 17 - exceeded max_slot_wal_keep_size
+    /can no longer get changes from replication slot/.test(message)
+  );
+}