@powersync/service-module-postgres 0.0.0-dev-20251111093449 → 0.0.0-dev-20251124070259

package/src/replication/WalStreamReplicationJob.ts

@@ -12,19 +12,13 @@ export interface WalStreamReplicationJobOptions extends replication.AbstractRepl
 
 export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
   private connectionFactory: ConnectionManagerFactory;
-  private readonly connectionManager: PgManager;
+  private connectionManager: PgManager | null = null;
   private lastStream: WalStream | null = null;
 
   constructor(options: WalStreamReplicationJobOptions) {
     super(options);
     this.logger = logger.child({ prefix: `[${this.slotName}] ` });
     this.connectionFactory = options.connectionFactory;
-    this.connectionManager = this.connectionFactory.create({
-      // Pool connections are only used intermittently.
-      idleTimeout: 30_000,
-      maxSize: 2,
-      applicationName: getApplicationName()
-    });
   }
 
   /**
@@ -40,10 +34,12 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
    * **This may be a bug in pgwire or how we're using it.
    */
   async keepAlive() {
-    try {
-      await sendKeepAlive(this.connectionManager.pool);
-    } catch (e) {
-      this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+    if (this.connectionManager) {
+      try {
+        await sendKeepAlive(this.connectionManager.pool);
+      } catch (e) {
+        this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
+      }
     }
   }
 
@@ -53,35 +49,58 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
 
   async replicate() {
     try {
-      await this.replicateLoop();
+      await this.replicateOnce();
     } catch (e) {
       // Fatal exception
-      container.reporter.captureException(e, {
-        metadata: {
-          replication_slot: this.slotName
+
+      if (!this.isStopped) {
+        // Ignore aborted errors
+
+        this.logger.error(`Replication error`, e);
+        if (e.cause != null) {
+          // Example:
+          // PgError.conn_ended: Unable to do postgres query on ended connection
+          //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
+          //     at stream.next (<anonymous>)
+          //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
+          //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
+          //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
+          //     ...
+          //   cause: TypeError: match is not iterable
+          //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
+          //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
+          //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
+          //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
+          //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
+          //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
+          //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
+          //   [Symbol(pg.ErrorCode)]: 'conn_ended',
+          //   [Symbol(pg.ErrorResponse)]: undefined
+          // }
+          // Without this additional log, the cause would not be visible in the logs.
+          this.logger.error(`cause`, e.cause);
         }
-      });
-      this.logger.error(`Replication failed`, e);
+        // Report the error if relevant, before retrying
+        container.reporter.captureException(e, {
+          metadata: {
+            replication_slot: this.slotName
+          }
+        });
+        // This sets the retry delay
+        this.rateLimiter.reportError(e);
+      }
 
       if (e instanceof MissingReplicationSlotError) {
         // This stops replication on this slot and restarts with a new slot
         await this.options.storage.factory.restartReplication(this.storage.group_id);
       }
+
+      // No need to rethrow - the error is already logged, and retry behavior is the same on error
     } finally {
       this.abortController.abort();
     }
   }
 
-  async replicateLoop() {
-    while (!this.isStopped) {
-      await this.replicateOnce();
-
-      if (!this.isStopped) {
-        await new Promise((resolve) => setTimeout(resolve, 5000));
-      }
-    }
-  }
-
   async replicateOnce() {
     // New connections on every iteration (every error with retry),
     // otherwise we risk repeating errors related to the connection,
@@ -92,6 +111,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
       maxSize: 2,
       applicationName: getApplicationName()
     });
+    this.connectionManager = connectionManager;
     try {
       await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
       if (this.isStopped) {
@@ -106,48 +126,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
       });
       this.lastStream = stream;
       await stream.replicate();
-    } catch (e) {
-      if (this.isStopped && e instanceof ReplicationAbortedError) {
-        // Ignore aborted errors
-        return;
-      }
-      this.logger.error(`Replication error`, e);
-      if (e.cause != null) {
-        // Example:
-        // PgError.conn_ended: Unable to do postgres query on ended connection
-        //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
-        //     at stream.next (<anonymous>)
-        //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
-        //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
-        //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
-        //     ...
-        //   cause: TypeError: match is not iterable
-        //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
-        //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
-        //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
-        //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
-        //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
-        //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
-        //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
-        //   [Symbol(pg.ErrorCode)]: 'conn_ended',
-        //   [Symbol(pg.ErrorResponse)]: undefined
-        // }
-        // Without this additional log, the cause would not be visible in the logs.
-        this.logger.error(`cause`, e.cause);
-      }
-      if (e instanceof MissingReplicationSlotError) {
-        throw e;
-      } else {
-        // Report the error if relevant, before retrying
-        container.reporter.captureException(e, {
-          metadata: {
-            replication_slot: this.slotName
-          }
-        });
-        // This sets the retry delay
-        this.rateLimiter?.reportError(e);
-      }
     } finally {
+      this.connectionManager = null;
       await connectionManager.end();
     }
   }

package/src/types/resolver.ts

@@ -1,8 +1,8 @@
-import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules';
 import * as pgwire from '@powersync/service-jpgwire';
-import { CustomTypeRegistry } from './registry.js';
+import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules';
 import semver from 'semver';
 import { getServerVersion } from '../utils/postgres_version.js';
+import { CustomTypeRegistry } from './registry.js';
 
 /**
  * Resolves descriptions used to decode values for custom postgres types.
@@ -11,11 +11,9 @@ import { getServerVersion } from '../utils/postgres_version.js';
  */
 export class PostgresTypeResolver {
   private cachedVersion: semver.SemVer | null = null;
+  readonly registry: CustomTypeRegistry;
 
-  constructor(
-    readonly registry: CustomTypeRegistry,
-    private readonly pool: pgwire.PgClient
-  ) {
+  constructor(private readonly pool: pgwire.PgClient) {
     this.registry = new CustomTypeRegistry();
   }
 
@@ -188,6 +186,11 @@ WHERE a.attnum > 0
     return toSyncRulesRow(record);
   }
 
+  constructRowRecord(columnMap: Record<string, number>, tupleRaw: Record<string, any>): SqliteInputRow {
+    const record = this.decodeTupleForTable(columnMap, tupleRaw);
+    return toSyncRulesRow(record);
+  }
+
   /**
    * We need a high level of control over how values are decoded, to make sure there is no loss
    * of precision in the process.
@@ -206,5 +209,23 @@ WHERE a.attnum > 0
     return result;
   }
 
+  /**
+   * We need a high level of control over how values are decoded, to make sure there is no loss
+   * of precision in the process.
+   */
+  private decodeTupleForTable(columnMap: Record<string, number>, tupleRaw: Record<string, any>): DatabaseInputRow {
+    let result: Record<string, any> = {};
+    for (let columnName in tupleRaw) {
+      const rawval = tupleRaw[columnName];
+      const typeOid = columnMap[columnName];
+      if (typeof rawval == 'string' && typeOid) {
+        result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid);
+      } else {
+        result[columnName] = rawval;
+      }
+    }
+    return result;
+  }
+
   private static minVersionForMultirange: semver.SemVer = semver.parse('14.0.0')!;
 }

package/src/types/types.ts

@@ -1,7 +1,6 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
-import { CustomTypeRegistry } from './registry.js';
 
 // Maintain backwards compatibility by exporting these
 export const validatePort = lib_postgres.validatePort;
@@ -25,10 +24,7 @@ export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig
 /**
  * Resolved version of {@link PostgresConnectionConfig}
  */
-export type ResolvedConnectionConfig = PostgresConnectionConfig &
-  NormalizedPostgresConnectionConfig & {
-    typeRegistry: CustomTypeRegistry;
-  };
+export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig;
 
 export function isPostgresConfig(
   config: service_types.configFile.DataSourceConfig

package/test/src/pg_test.test.ts

@@ -551,7 +551,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
   test('test replication - multiranges', async () => {
     const db = await connectPgPool();
 
-    if (!(await new PostgresTypeResolver(new CustomTypeRegistry(), db).supportsMultiRanges())) {
+    if (!(await new PostgresTypeResolver(db).supportsMultiRanges())) {
       // This test requires Postgres 14 or later.
       return;
     }
@@ -620,7 +620,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
  * Return all the inserts from the first transaction in the replication stream.
  */
 async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) {
-  const typeCache = new PostgresTypeResolver(new CustomTypeRegistry(), db);
+  const typeCache = new PostgresTypeResolver(db);
   await typeCache.fetchTypesForSchema();
 
   let transformed: SqliteInputRow[] = [];

package/test/src/slow_tests.test.ts

@@ -69,7 +69,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
   });
 
   async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
-    const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
+    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
     const replicationConnection = await connections.replicationConnection();
     const pool = connections.pool;
     await clearTestDb(pool);
@@ -330,7 +330,7 @@ bucket_definitions:
       await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
       i += 1;
 
-      const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
+      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
       const replicationConnection = await connections.replicationConnection();
 
       abortController = new AbortController();

package/test/src/wal_stream.test.ts

@@ -529,13 +529,24 @@ config:
     const { pool } = context;
     await pool.query(`DROP TABLE IF EXISTS test_data`);
     await pool.query(`CREATE TYPE composite AS (foo bool, bar int4);`);
-    await pool.query(`CREATE TABLE test_data(id text primary key, description composite);`);
+    await pool.query(`CREATE TABLE test_data(id text primary key, description composite, ts timestamptz);`);
+
+    // Covered by initial replication
+    await pool.query(
+      `INSERT INTO test_data(id, description, ts) VALUES ('t1', ROW(TRUE, 1)::composite, '2025-11-17T09:11:00Z')`
+    );
 
     await context.initializeReplication();
-    await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`);
+    // Covered by streaming replication
+    await pool.query(
+      `INSERT INTO test_data(id, description, ts) VALUES ('t2', ROW(TRUE, 2)::composite, '2025-11-17T09:12:00Z')`
+    );
 
     const data = await context.getBucketData('1#stream|0[]');
-    expect(data).toMatchObject([putOp('test_data', { id: 't1', description: '{"foo":1,"bar":2}' })]);
+    expect(data).toMatchObject([
+      putOp('test_data', { id: 't1', description: '{"foo":1,"bar":1}', ts: '2025-11-17T09:11:00.000000Z' }),
+      putOp('test_data', { id: 't2', description: '{"foo":1,"bar":2}', ts: '2025-11-17T09:12:00.000000Z' })
+    ]);
   });
 
   test('custom types in primary key', async () => {

package/test/src/wal_stream_utils.ts

@@ -33,7 +33,7 @@ export class WalStreamTestContext implements AsyncDisposable {
     options?: { doNotClear?: boolean; walStreamOptions?: Partial<WalStreamOptions> }
   ) {
     const f = await factory({ doNotClear: options?.doNotClear });
-    const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
+    const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
 
     if (!options?.doNotClear) {
       await clearTestDb(connectionManager.pool);