@powersync/service-module-postgres 0.13.1 → 0.14.1

package/test/src/resuming_snapshots.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { describeWithStorage } from './util.js';
+import { WalStreamTestContext } from './wal_stream_utils.js';
+import { TestStorageFactory } from '@powersync/service-core';
+import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { ReplicationMetric } from '@powersync/service-types';
+import * as timers from 'node:timers/promises';
+import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+
+describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
+  describeWithStorage({ timeout: 240_000 }, function (factory) {
+    test('resuming initial replication (1)', async () => {
+      // Stop early - likely to not include deleted row in first replication attempt.
+      await testResumingReplication(factory, 2000);
+    });
+    test('resuming initial replication (2)', async () => {
+      // Stop late - likely to include deleted row in first replication attempt.
+      await testResumingReplication(factory, 8000);
+    });
+  });
+});
+
+async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
+  // This tests interrupting and then resuming initial replication.
+  // We interrupt replication after test_data1 has fully replicated, and
+  // test_data2 has partially replicated.
+  // This test relies on interval behavior that is not 100% deterministic:
+  // 1. We attempt to abort initial replication once a certain number of
+  //    rows have been replicated, but this is not exact. Our only requirement
+  //    is that we have not fully replicated test_data2 yet.
+  // 2. Order of replication is not deterministic, so which specific rows
+  //    have been / have not been replicated at that point is not deterministic.
+  //    We do allow for some variation in the test results to account for this.
+
+  await using context = await WalStreamTestContext.open(factory, { walStreamOptions: { snapshotChunkLength: 1000 } });
+
+  await context.updateSyncRules(`bucket_definitions:
+  global:
+    data:
+      - SELECT * FROM test_data1
+      - SELECT * FROM test_data2`);
+  const { pool } = context;
+
+  await pool.query(`CREATE TABLE test_data1(id serial primary key, description text)`);
+  await pool.query(`CREATE TABLE test_data2(id serial primary key, description text)`);
+
+  await pool.query(
+    {
+      statement: `INSERT INTO test_data1(description) SELECT 'foo' FROM generate_series(1, 1000) i`
+    },
+    {
+      statement: `INSERT INTO test_data2( description) SELECT 'foo' FROM generate_series(1, 10000) i`
+    }
+  );
+
+  const p = context.replicateSnapshot();
+
+  let done = false;
+
+  const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+  try {
+    (async () => {
+      while (!done) {
+        const count =
+          ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
+
+        if (count >= stopAfter) {
+          break;
+        }
+        await timers.setTimeout(1);
+      }
+      // This interrupts initial replication
+      await context.dispose();
+    })();
+    // This confirms that initial replication was interrupted
+    const error = await p.catch((e) => e);
+    expect(error).toBeInstanceOf(ReplicationAbortedError);
+    done = true;
+  } finally {
+    done = true;
+  }
+
+  // Bypass the usual "clear db on factory open" step.
+  await using context2 = await WalStreamTestContext.open(factory, {
+    doNotClear: true,
+    walStreamOptions: { snapshotChunkLength: 1000 }
+  });
+
+  // This delete should be using one of the ids already replicated
+  const {
+    rows: [[id1]]
+  } = await context2.pool.query(`DELETE FROM test_data2 WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`);
+  // This update should also be using one of the ids already replicated
+  const {
+    rows: [[id2]]
+  } = await context2.pool.query(
+    `UPDATE test_data2 SET description = 'update1' WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`
+  );
+  const {
+    rows: [[id3]]
+  } = await context2.pool.query(`INSERT INTO test_data2(description) SELECT 'insert1' RETURNING id`);
+
+  await context2.loadNextSyncRules();
+  await context2.replicateSnapshot();
+
+  context2.startStreaming();
+  const data = await context2.getBucketData('global[]', undefined, {});
+
+  const deletedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id1));
+  const updatedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id2));
+  const insertedRowOps = data.filter((row) => row.object_type == 'test_data2' && row.object_id === String(id3));
+
+  if (deletedRowOps.length != 0) {
+    // The deleted row was part of the first replication batch,
+    // so it is removed by streaming replication.
+    expect(deletedRowOps.length).toEqual(2);
+    expect(deletedRowOps[1].op).toEqual('REMOVE');
+  } else {
+    // The deleted row was not part of the first replication batch,
+    // so it's not in the resulting ops at all.
+  }
+
+  expect(updatedRowOps.length).toEqual(2);
+  // description for the first op could be 'foo' or 'update1'.
+  // We only test the final version.
+  expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
+
+  expect(insertedRowOps.length).toEqual(2);
+  expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
+  expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+
+  // 1000 of test_data1 during first replication attempt.
+  // N >= 1000 of test_data2 during first replication attempt.
+  // 10000 - N - 1 + 1 of test_data2 during second replication attempt.
+  // An additional update during streaming replication (2x total for this row).
+  // An additional insert during streaming replication (2x total for this row).
+  // If the deleted row was part of the first replication batch, it's removed by streaming replication.
+  // This adds 2 ops.
+  // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
+  // However, this is not deterministic.
+  const expectedCount = 11002 + deletedRowOps.length;
+  expect(data.length).toEqual(expectedCount);
+
+  const replicatedCount =
+    ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
+
+  // With resumable replication, there should be no need to re-replicate anything.
+  expect(replicatedCount).toEqual(expectedCount);
+}

package/test/src/schema_changes.test.ts

@@ -3,16 +3,11 @@ import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 
 import { storage } from '@powersync/service-core';
-import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
+import { describeWithStorage } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
-describe.skipIf(!env.TEST_MONGO_STORAGE)('schema changes - mongodb', { timeout: 20_000 }, function () {
-  defineTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-});
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('schema changes - postgres', { timeout: 20_000 }, function () {
-  defineTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+describe('schema changes', { timeout: 20_000 }, function () {
+  describeWithStorage({}, defineTests);
 });
 
 const BASIC_SYNC_RULES = `
@@ -459,7 +454,7 @@ function defineTests(factory: storage.TestStorageFactory) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
     // Just as an FYI - cannot update or delete here
-    expect(pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = 't2'`)).rejects.toThrow(
+    await expect(pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = 't2'`)).rejects.toThrow(
       'does not have a replica identity and publishes updates'
     );
 
@@ -497,7 +492,7 @@ function defineTests(factory: storage.TestStorageFactory) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
     // Just as an FYI - cannot update or delete here
-    expect(pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = 't2'`)).rejects.toThrow(
+    await expect(pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = 't2'`)).rejects.toThrow(
       'does not have a replica identity and publishes updates'
     );
 

package/test/src/slow_tests.test.ts

@@ -5,9 +5,8 @@ import { env } from './env.js';
 import {
   clearTestDb,
   connectPgPool,
+  describeWithStorage,
   getClientCheckpoint,
-  INITIALIZED_MONGO_STORAGE_FACTORY,
-  INITIALIZED_POSTGRES_STORAGE_FACTORY,
   TEST_CONNECTION_OPTIONS
 } from './util.js';
 
@@ -21,26 +20,10 @@ import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import * as timers from 'node:timers/promises';
 
-describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () {
-  // These are slow, inconsistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
-});
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('slow tests - postgres', function () {
-  // These are slow, inconsistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
+describe.skipIf(!(env.CI || env.SLOW_TESTS))('slow tests', function () {
+  describeWithStorage({ timeout: 120_000 }, function (factory) {
+    defineSlowTests(factory);
+  });
 });
 
 function defineSlowTests(factory: storage.TestStorageFactory) {
@@ -350,14 +333,14 @@ bucket_definitions:
       const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
       const replicationConnection = await connections.replicationConnection();
 
-        abortController = new AbortController();
-        const options: WalStreamOptions = {
-          abort_signal: abortController.signal,
-          connections,
-          storage: storage,
-          metrics: METRICS_HELPER.metricsEngine
-        };
-        walStream = new WalStream(options);
+      abortController = new AbortController();
+      const options: WalStreamOptions = {
+        abort_signal: abortController.signal,
+        connections,
+        storage: storage,
+        metrics: METRICS_HELPER.metricsEngine
+      };
+      walStream = new WalStream(options);
 
       await storage.clear();
 

package/test/src/util.ts

@@ -2,11 +2,12 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'
 import * as types from '@module/types/types.js';
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { logger } from '@powersync/lib-services-framework';
-import { BucketStorageFactory, InternalOpId, TestStorageOptions } from '@powersync/service-core';
+import { BucketStorageFactory, InternalOpId, TestStorageFactory, TestStorageOptions } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import { env } from './env.js';
+import { describe, TestOptions } from 'vitest';
 
 export const TEST_URI = env.PG_TEST_URL;
 
@@ -19,6 +20,16 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTes
   url: env.PG_STORAGE_TEST_URL
 });
 
+export function describeWithStorage(options: TestOptions, fn: (factory: TestStorageFactory) => void) {
+  describe.skipIf(!env.TEST_MONGO_STORAGE)(`mongodb storage`, options, function () {
+    fn(INITIALIZED_MONGO_STORAGE_FACTORY);
+  });
+
+  describe.skipIf(!env.TEST_POSTGRES_STORAGE)(`postgres storage`, options, function () {
+    fn(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  });
+}
+
 export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   type: 'postgresql',
   uri: TEST_URI,

package/test/src/validation.test.ts

@@ -1,7 +1,6 @@
 import { getDebugTablesInfo } from '@module/replication/replication-utils.js';
 import { expect, test } from 'vitest';
 
-// Not quite a walStreamTest, but it helps to manage the connection
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 

package/test/src/wal_stream.test.ts

@@ -2,12 +2,11 @@ import { MissingReplicationSlotError } from '@module/replication/WalStream.js';
 import { storage } from '@powersync/service-core';
 import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
 import { pgwireRows } from '@powersync/service-jpgwire';
+import { ReplicationMetric } from '@powersync/service-types';
 import * as crypto from 'crypto';
 import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
+import { describeWithStorage } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
-import { ReplicationMetric } from '@powersync/service-types';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -16,12 +15,8 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
 
-describe.skipIf(!env.TEST_MONGO_STORAGE)('wal stream - mongodb', { timeout: 20_000 }, function () {
-  defineWalStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-});
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)('wal stream - postgres', { timeout: 20_000 }, function () {
-  defineWalStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+describe('wal stream', () => {
+  describeWithStorage({ timeout: 20_000 }, defineWalStreamTests);
 });
 
 function defineWalStreamTests(factory: storage.TestStorageFactory) {

package/test/src/wal_stream_utils.ts

@@ -19,6 +19,7 @@ export class WalStreamTestContext implements AsyncDisposable {
   private streamPromise?: Promise<void>;
   public storage?: SyncRulesBucketStorage;
   private replicationConnection?: pgwire.PgConnection;
+  private snapshotPromise?: Promise<void>;
 
   /**
    * Tests operating on the wal stream need to configure the stream and manage asynchronous
@@ -28,7 +29,7 @@ export class WalStreamTestContext implements AsyncDisposable {
    */
   static async open(
     factory: (options: storage.TestStorageOptions) => Promise<BucketStorageFactory>,
-    options?: { doNotClear?: boolean }
+    options?: { doNotClear?: boolean; walStreamOptions?: Partial<WalStreamOptions> }
   ) {
     const f = await factory({ doNotClear: options?.doNotClear });
     const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
@@ -37,12 +38,13 @@ export class WalStreamTestContext implements AsyncDisposable {
       await clearTestDb(connectionManager.pool);
     }
 
-    return new WalStreamTestContext(f, connectionManager);
+    return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions);
   }
 
   constructor(
     public factory: BucketStorageFactory,
-    public connectionManager: PgManager
+    public connectionManager: PgManager,
+    private walStreamOptions?: Partial<WalStreamOptions>
   ) {
     createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
     initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -54,6 +56,7 @@ export class WalStreamTestContext implements AsyncDisposable {
 
   async dispose() {
     this.abortController.abort();
+    await this.snapshotPromise;
     await this.streamPromise;
     await this.connectionManager.destroy();
     await this.factory?.[Symbol.asyncDispose]();
@@ -108,16 +111,21 @@ export class WalStreamTestContext implements AsyncDisposable {
       storage: this.storage,
       metrics: METRICS_HELPER.metricsEngine,
       connections: this.connectionManager,
-      abort_signal: this.abortController.signal
+      abort_signal: this.abortController.signal,
+      ...this.walStreamOptions
     };
     this._walStream = new WalStream(options);
     return this._walStream!;
   }
 
   async replicateSnapshot() {
-    this.replicationConnection = await this.connectionManager.replicationConnection();
-    await this.walStream.initReplication(this.replicationConnection);
-    await this.storage!.autoActivate();
+    const promise = (async () => {
+      this.replicationConnection = await this.connectionManager.replicationConnection();
+      await this.walStream.initReplication(this.replicationConnection);
+      await this.storage!.autoActivate();
+    })();
+    this.snapshotPromise = promise.catch((e) => e);
+    await promise;
   }
 
   startStreaming() {