@powersync/service-module-postgres 0.0.0-dev-20241007145127 → 0.0.0-dev-20241015210820

package/src/replication/WalStream.ts

@@ -1,11 +1,11 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import * as util from '../utils/pgwire_utils.js';
 import { container, errors, logger } from '@powersync/lib-services-framework';
+import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
+import * as pgwire from '@powersync/service-jpgwire';
 import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
+import * as pg_utils from '../utils/pgwire_utils.js';
+import { PgManager } from './PgManager.js';
 import { getPgOutputRelation, getRelId } from './PgRelation.js';
-import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
 import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js';
-import { PgManager } from './PgManager.js';
 
 export const ZERO_LSN = '00000000/00000000';
 export const PUBLICATION_NAME = 'powersync';
@@ -60,7 +60,7 @@ export class WalStream {
           // Ping to speed up cancellation of streaming replication
           // We're not using pg_snapshot here, since it could be in the middle of
           // an initial replication transaction.
-          const promise = util.retriedQuery(
+          const promise = pg_utils.retriedQuery(
             this.connections.pool,
             `SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`
           );
@@ -347,7 +347,6 @@ WHERE  oid = $1::regclass`,
         for (let table of tables) {
           await this.snapshotTable(batch, db, table);
           await batch.markSnapshotDone([table], lsn);
-
           await touch();
         }
       }
@@ -395,10 +394,10 @@ WHERE  oid = $1::regclass`,
         throw new Error(`Aborted initial replication of ${this.slot_name}`);
       }
 
-      for (let record of WalStream.getQueryData(rows)) {
+      for (const record of WalStream.getQueryData(rows)) {
         // This auto-flushes when the batch reaches its size limit
         await batch.save({
-          tag: 'insert',
+          tag: storage.SaveOperationTag.INSERT,
           sourceTable: table,
           before: undefined,
           beforeReplicaId: undefined,
@@ -406,6 +405,7 @@ WHERE  oid = $1::regclass`,
           afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns)
         });
       }
+
       at += rows.length;
       Metrics.getInstance().rows_replicated_total.add(rows.length);
 
@@ -495,9 +495,9 @@ WHERE  oid = $1::regclass`,
 
       if (msg.tag == 'insert') {
         Metrics.getInstance().rows_replicated_total.add(1);
-        const baseRecord = util.constructAfterRecord(msg);
+        const baseRecord = pg_utils.constructAfterRecord(msg);
         return await batch.save({
-          tag: 'insert',
+          tag: storage.SaveOperationTag.INSERT,
           sourceTable: table,
           before: undefined,
           beforeReplicaId: undefined,
@@ -508,10 +508,10 @@ WHERE  oid = $1::regclass`,
         Metrics.getInstance().rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
-        const before = util.constructBeforeRecord(msg);
-        const after = util.constructAfterRecord(msg);
+        const before = pg_utils.constructBeforeRecord(msg);
+        const after = pg_utils.constructAfterRecord(msg);
         return await batch.save({
-          tag: 'update',
+          tag: storage.SaveOperationTag.UPDATE,
           sourceTable: table,
           before: before,
           beforeReplicaId: before ? getUuidReplicaIdentityBson(before, table.replicaIdColumns) : undefined,
@@ -520,10 +520,10 @@ WHERE  oid = $1::regclass`,
         });
       } else if (msg.tag == 'delete') {
         Metrics.getInstance().rows_replicated_total.add(1);
-        const before = util.constructBeforeRecord(msg)!;
+        const before = pg_utils.constructBeforeRecord(msg)!;
 
         return await batch.save({
-          tag: 'delete',
+          tag: storage.SaveOperationTag.DELETE,
           sourceTable: table,
           before: before,
           beforeReplicaId: getUuidReplicaIdentityBson(before, table.replicaIdColumns),
@@ -592,7 +592,6 @@ WHERE  oid = $1::regclass`,
         // chunkLastLsn may come from normal messages in the chunk,
         // or from a PrimaryKeepalive message.
         const { messages, lastLsn: chunkLastLsn } = chunk;
-
         for (const msg of messages) {
           if (msg.tag == 'relation') {
             await this.handleRelation(batch, getPgOutputRelation(msg), true);
@@ -609,7 +608,7 @@ WHERE  oid = $1::regclass`,
             }
 
             count += 1;
-            const result = await this.writeChange(batch, msg);
+            await this.writeChange(batch, msg);
           }
         }
 

package/src/replication/WalStreamReplicationJob.ts

@@ -1,6 +1,6 @@
-import { MissingReplicationSlotError, WalStream } from './WalStream.js';
 import { container } from '@powersync/lib-services-framework';
 import { PgManager } from './PgManager.js';
+import { MissingReplicationSlotError, WalStream } from './WalStream.js';
 
 import { replication } from '@powersync/service-core';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';

package/src/replication/WalStreamReplicator.ts

@@ -1,7 +1,7 @@
-import { storage, replication } from '@powersync/service-core';
-import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
+import { replication, storage } from '@powersync/service-core';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
 import { cleanUpReplicationSlot } from './replication-utils.js';
+import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
 
 export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;

package/src/types/types.ts

@@ -22,7 +22,7 @@ export interface NormalizedPostgresConnectionConfig {
   client_private_key: string | undefined;
 }
 
-export const PostgresConnectionConfig = service_types.configFile.dataSourceConfig.and(
+export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and(
   t.object({
     type: t.literal(POSTGRES_CONNECTION_TYPE),
     /** Unique identifier for the connection - optional when a single connection is present. */

package/test/src/slow_tests.test.ts

@@ -2,16 +2,16 @@ import * as bson from 'bson';
 import { afterEach, describe, expect, test } from 'vitest';
 import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
 import { env } from './env.js';
-import { TEST_CONNECTION_OPTIONS, clearTestDb, connectPgPool, getClientCheckpoint } from './util.js';
+import { clearTestDb, connectPgPool, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
 
 import * as pgwire from '@powersync/service-jpgwire';
 import { SqliteRow } from '@powersync/service-sync-rules';
 
 import { mapOpEntry, MongoBucketStorage } from '@/storage/storage-index.js';
-import * as timers from 'node:timers/promises';
+import { reduceBucket, validateCompactedBucket } from '@core-tests/bucket_validation.js';
 import { MONGO_STORAGE_FACTORY, StorageFactory } from '@core-tests/util.js';
 import { PgManager } from '@module/replication/PgManager.js';
-import { reduceBucket, validateCompactedBucket } from '@core-tests/bucket_validation.js';
+import * as timers from 'node:timers/promises';
 
 describe('slow tests - mongodb', function () {
   // These are slow, inconsistent tests.
@@ -82,7 +82,7 @@ bucket_definitions:
       - SELECT * FROM "test_data"
 `;
     const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-    const storage = f.getInstance(syncRules);
+    using storage = f.getInstance(syncRules);
     abortController = new AbortController();
     const options: WalStreamOptions = {
       abort_signal: abortController.signal,
@@ -234,7 +234,7 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
       const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-      const storage = f.getInstance(syncRules);
+      using storage = f.getInstance(syncRules);
 
       // 1. Setup some base data that will be replicated in initial replication
       await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);

package/test/src/util.ts

@@ -1,11 +1,11 @@
+import { connectMongo } from '@core-tests/util.js';
 import * as types from '@module/types/types.js';
 import * as pg_utils from '@module/utils/pgwire_utils.js';
+import { logger } from '@powersync/lib-services-framework';
 import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
-import { env } from './env.js';
 import { pgwireRows } from '@powersync/service-jpgwire';
-import { logger } from '@powersync/lib-services-framework';
-import { connectMongo } from '@core-tests/util.js';
+import { env } from './env.js';
 
 // The metrics need to be initialized before they can be used
 await Metrics.initialise({
@@ -35,7 +35,9 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
 
   await db.clear();
 
-  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
+  return new MongoBucketStorage(db, {
+    slot_name_prefix: 'test_'
+  });
 };
 
 export async function clearTestDb(db: pgwire.PgClient) {

package/test/src/wal_stream_utils.ts

@@ -1,9 +1,9 @@
-import { BucketStorageFactory, SyncRulesBucketStorage } from '@powersync/service-core';
-import * as pgwire from '@powersync/service-jpgwire';
-import { TEST_CONNECTION_OPTIONS, clearTestDb, getClientCheckpoint } from './util.js';
-import { WalStream, WalStreamOptions, PUBLICATION_NAME } from '@module/replication/WalStream.js';
 import { fromAsync } from '@core-tests/stream_utils.js';
 import { PgManager } from '@module/replication/PgManager.js';
+import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js';
+import { BucketStorageFactory, SyncRulesBucketStorage } from '@powersync/service-core';
+import * as pgwire from '@powersync/service-jpgwire';
+import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
 
 /**
  * Tests operating on the wal stream need to configure the stream and manage asynchronous
@@ -20,16 +20,12 @@ export function walStreamTest(
     const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
 
     await clearTestDb(connectionManager.pool);
-    const context = new WalStreamTestContext(f, connectionManager);
-    try {
-      await test(context);
-    } finally {
-      await context.dispose();
-    }
+    await using context = new WalStreamTestContext(f, connectionManager);
+    await test(context);
   };
 }
 
-export class WalStreamTestContext {
+export class WalStreamTestContext implements AsyncDisposable {
   private _walStream?: WalStream;
   private abortController = new AbortController();
   private streamPromise?: Promise<void>;
@@ -41,10 +37,11 @@ export class WalStreamTestContext {
     public connectionManager: PgManager
   ) {}
 
-  async dispose() {
+  async [Symbol.asyncDispose]() {
     this.abortController.abort();
     await this.streamPromise;
     await this.connectionManager.destroy();
+    this.storage?.[Symbol.dispose]();
   }
 
   get pool() {