@powersync/service-module-postgres 0.16.10 → 0.16.11

package/src/replication/WalStreamReplicationJob.ts

@@ -12,19 +12,13 @@ export interface WalStreamReplicationJobOptions extends replication.AbstractRepl
 
 export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
   private connectionFactory: ConnectionManagerFactory;
-  private readonly connectionManager: PgManager;
+  private connectionManager: PgManager | null = null;
   private lastStream: WalStream | null = null;
 
   constructor(options: WalStreamReplicationJobOptions) {
     super(options);
     this.logger = logger.child({ prefix: `[${this.slotName}] ` });
     this.connectionFactory = options.connectionFactory;
-    this.connectionManager = this.connectionFactory.create({
-      // Pool connections are only used intermittently.
-      idleTimeout: 30_000,
-      maxSize: 2,
-      applicationName: getApplicationName()
-    });
   }
 
   /**
@@ -40,10 +34,12 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
    * **This may be a bug in pgwire or how we're using it.
    */
   async keepAlive() {
-    try {
-      await sendKeepAlive(this.connectionManager.pool);
-    } catch (e) {
-      this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+    if (this.connectionManager) {
+      try {
+        await sendKeepAlive(this.connectionManager.pool);
+      } catch (e) {
+        this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+      }
     }
   }
 
@@ -53,35 +49,58 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
 
   async replicate() {
     try {
-      await this.replicateLoop();
+      await this.replicateOnce();
     } catch (e) {
       // Fatal exception
-      container.reporter.captureException(e, {
-        metadata: {
-          replication_slot: this.slotName
+
+      if (!this.isStopped) {
+        // Ignore aborted errors
+
+        this.logger.error(`Replication error`, e);
+        if (e.cause != null) {
+          // Example:
+          // PgError.conn_ended: Unable to do postgres query on ended connection
+          //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
+          //     at stream.next (<anonymous>)
+          //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
+          //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
+          //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
+          //     ...
+          //   cause: TypeError: match is not iterable
+          //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
+          //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
+          //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
+          //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
+          //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
+          //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
+          //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
+          //   [Symbol(pg.ErrorCode)]: 'conn_ended',
+          //   [Symbol(pg.ErrorResponse)]: undefined
+          // }
+          // Without this additional log, the cause would not be visible in the logs.
+          this.logger.error(`cause`, e.cause);
         }
-      });
-      this.logger.error(`Replication failed`, e);
+        // Report the error if relevant, before retrying
+        container.reporter.captureException(e, {
+          metadata: {
+            replication_slot: this.slotName
+          }
+        });
+        // This sets the retry delay
+        this.rateLimiter.reportError(e);
+      }
 
       if (e instanceof MissingReplicationSlotError) {
         // This stops replication on this slot and restarts with a new slot
         await this.options.storage.factory.restartReplication(this.storage.group_id);
       }
+
+      // No need to rethrow - the error is already logged, and retry behavior is the same on error
     } finally {
       this.abortController.abort();
     }
   }
 
-  async replicateLoop() {
-    while (!this.isStopped) {
-      await this.replicateOnce();
-
-      if (!this.isStopped) {
-        await new Promise((resolve) => setTimeout(resolve, 5000));
-      }
-    }
-  }
-
   async replicateOnce() {
     // New connections on every iteration (every error with retry),
     // otherwise we risk repeating errors related to the connection,
@@ -92,6 +111,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
       maxSize: 2,
       applicationName: getApplicationName()
     });
+    this.connectionManager = connectionManager;
     try {
       await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
       if (this.isStopped) {
@@ -106,48 +126,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
       });
       this.lastStream = stream;
       await stream.replicate();
-    } catch (e) {
-      if (this.isStopped && e instanceof ReplicationAbortedError) {
-        // Ignore aborted errors
-        return;
-      }
-      this.logger.error(`Replication error`, e);
-      if (e.cause != null) {
-        // Example:
-        // PgError.conn_ended: Unable to do postgres query on ended connection
-        //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-        //     at stream.next (<anonymous>)
-        //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-        //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-        //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-        //     ...
-        //   cause: TypeError: match is not iterable
-        //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-        //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-        //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-        //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-        //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-        //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-        //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-        //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-        //   [Symbol(pg.ErrorResponse)]: undefined
-        // }
-        // Without this additional log, the cause would not be visible in the logs.
-        this.logger.error(`cause`, e.cause);
-      }
-      if (e instanceof MissingReplicationSlotError) {
-        throw e;
-      } else {
-        // Report the error if relevant, before retrying
-        container.reporter.captureException(e, {
-          metadata: {
-            replication_slot: this.slotName
-          }
-        });
-        // This sets the retry delay
-        this.rateLimiter?.reportError(e);
-      }
     } finally {
+      this.connectionManager = null;
       await connectionManager.end();
     }
   }

package/test/src/wal_stream.test.ts

@@ -295,7 +295,7 @@ bucket_definitions:
         `INSERT INTO test_data(id, description) VALUES('8133cd37-903b-4937-a022-7c8294015a3a', 'test1') returning id as test_id`
       );
       await context.replicateSnapshot();
-      await context.startStreaming();
+      context.startStreaming();
 
       const data = await context.getBucketData('global[]');
 
@@ -320,17 +320,25 @@ bucket_definitions:
 
       await context.loadActiveSyncRules();
 
+      // Previously, the `replicateSnapshot` call picked up on this error.
+      // Now, we have removed that check, this only comes up when we start actually streaming.
+      // We don't get the streaming response directly here, but getCheckpoint() checks for that.
+      await context.replicateSnapshot();
+      context.startStreaming();
+
       if (serverVersion!.compareMain('18.0.0') >= 0) {
-        await context.replicateSnapshot();
         // No error expected in Postres 18. Replication keeps on working depite the
         // publication being re-created.
+        await context.getCheckpoint();
       } else {
+        // await context.getCheckpoint();
         // Postgres < 18 invalidates the replication slot when the publication is re-created.
-        // The error is handled on a higher level, which triggers
+        // In the service, this error is handled in WalStreamReplicationJob,
         // creating a new replication slot.
         await expect(async () => {
-          await context.replicateSnapshot();
+          await context.getCheckpoint();
         }).rejects.toThrowError(MissingReplicationSlotError);
+        context.clearStreamError();
       }
     }
   });
@@ -352,7 +360,7 @@ bucket_definitions:
         `INSERT INTO test_data(id, description) VALUES('8133cd37-903b-4937-a022-7c8294015a3a', 'test1') returning id as test_id`
       );
       await context.replicateSnapshot();
-      await context.startStreaming();
+      context.startStreaming();
 
       const data = await context.getBucketData('global[]');
 
@@ -415,7 +423,7 @@ bucket_definitions:
         `INSERT INTO test_data(id, description) VALUES('8133cd37-903b-4937-a022-7c8294015a3a', 'test1') returning id as test_id`
       );
       await context.replicateSnapshot();
-      await context.startStreaming();
+      context.startStreaming();
 
       const data = await context.getBucketData('global[]');
 
@@ -572,7 +580,7 @@ config:
     );
 
     await context.replicateSnapshot();
-    await context.startStreaming();
+    context.startStreaming();
 
     await pool.query(`UPDATE test_data SET description = 'test2' WHERE id = '${test_id}'`);
 

package/test/src/wal_stream_utils.ts

@@ -55,12 +55,31 @@ export class WalStreamTestContext implements AsyncDisposable {
     await this.dispose();
   }
 
+  /**
+   * Clear any errors from startStream, to allow for a graceful dispose when streaming errors
+   * were expected.
+   */
+  async clearStreamError() {
+    if (this.streamPromise != null) {
+      this.streamPromise = this.streamPromise.catch((e) => {});
+    }
+  }
+
   async dispose() {
     this.abortController.abort();
-    await this.snapshotPromise;
-    await this.streamPromise;
-    await this.connectionManager.destroy();
-    await this.factory?.[Symbol.asyncDispose]();
+    try {
+      await this.snapshotPromise;
+      await this.streamPromise;
+      await this.connectionManager.destroy();
+      await this.factory?.[Symbol.asyncDispose]();
+    } catch (e) {
+      // Throwing here may result in SuppressedError. The underlying errors often don't show up
+      // in the test output, so we log it here.
+      // If we could get vitest to log SuppressedError.error and SuppressedError.suppressed, we
+      // could remove this.
+      console.error('Error during WalStreamTestContext dispose', e);
+      throw e;
+    }
   }
 
   get pool() {