@powersync/service-module-postgres 0.17.2 → 0.19.0

package/test/src/chunked_snapshots.test.ts

@@ -1,17 +1,21 @@
-import { reduceBucket, TestStorageFactory } from '@powersync/service-core';
+import { reduceBucket } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as crypto from 'node:crypto';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
-import { describeWithStorage } from './util.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
 describe('chunked snapshots', () => {
   describeWithStorage({ timeout: 120_000 }, defineBatchTests);
 });
 
-function defineBatchTests(factory: TestStorageFactory) {
+function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
+    return WalStreamTestContext.open(factory, { ...options, storageVersion });
+  };
+
   // We need to test every supported type, since chunking could be quite sensitive to
   // how each specific type is handled.
   test('chunked snapshot edge case (int2)', async () => {
@@ -89,7 +93,7 @@ function defineBatchTests(factory: TestStorageFactory) {
     // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
     // 6. We end up with a row that has a missing TOAST column.
 
-    await using context = await WalStreamTestContext.open(factory, {
+    await using context = await openContext({
       // We need to use a smaller chunk size here, so that we can run a query in between chunks
       walStreamOptions: { snapshotChunkLength: 100 }
     });
@@ -142,7 +146,8 @@ function defineBatchTests(factory: TestStorageFactory) {
     await p;
 
     // 5. Logical replication picks up the UPDATE above, but it is missing the TOAST column.
-    context.startStreaming();
+    // Note: logical replication now runs concurrently with the snapshot.
+    // TODO: re-check the test logic here.
 
     // 6. If all went well, the "resnapshot" process would take care of this.
     const data = await context.getBucketData('global[]', undefined, {});

package/test/src/large_batch.test.ts

@@ -1,14 +1,11 @@
-import { storage } from '@powersync/service-core';
 import { describe, expect, test } from 'vitest';
 import { populateData } from '../../dist/utils/populate_test_data.js';
 import { env } from './env.js';
-import { describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js';
+import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, function (factory) {
-    defineBatchTests(factory);
-  });
+  describeWithStorage({ timeout: 240_000 }, defineBatchTests);
 });
 
 const BASIC_SYNC_RULES = `bucket_definitions:
@@ -16,9 +13,13 @@ const BASIC_SYNC_RULES = `bucket_definitions:
     data:
       - SELECT id, description, other FROM "test_data"`;
 
-function defineBatchTests(factory: storage.TestStorageFactory) {
+function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
+    return WalStreamTestContext.open(factory, { ...options, storageVersion });
+  };
+
   test('update large record', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     // This test generates a large transaction in MongoDB, despite the replicated data
     // not being that large.
     // If we don't limit transaction size, we could run into this error:
@@ -39,19 +40,16 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
 
     const start = Date.now();
 
-    context.startStreaming();
-
-    const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+    const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
     const duration = Date.now() - start;
     const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-    const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
     expect(checksum.get('global[]')!.count).toEqual(operation_count);
     const perSecond = Math.round((operation_count / duration) * 1000);
     console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
   });
 
   test('initial replication performance', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     // Manual test to check initial replication performance and memory usage
     await context.updateSyncRules(BASIC_SYNC_RULES);
     const { pool } = context;
@@ -87,11 +85,9 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
       const start = Date.now();
 
       await context.replicateSnapshot();
-      context.startStreaming();
 
-      const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+      const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
       const duration = Date.now() - start;
-      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
       expect(checksum.get('global[]')!.count).toEqual(operation_count);
       const perSecond = Math.round((operation_count / duration) * 1000);
       console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
@@ -102,7 +98,7 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
   });
 
   test('large number of operations', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     // This just tests performance of a large number of operations inside a transaction.
     await context.updateSyncRules(BASIC_SYNC_RULES);
     const { pool } = context;
@@ -139,12 +135,9 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
 
     const start = Date.now();
 
-    context.startStreaming();
-
-    const checkpoint = await context.getCheckpoint({ timeout: 50_000 });
+    const checksum = await context.getChecksums(['global[]']);
     const duration = Date.now() - start;
     const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-    const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
     expect(checksum.get('global[]')!.count).toEqual(operationCount);
     const perSecond = Math.round((operationCount / duration) * 1000);
     // This number depends on the test machine, so we keep the test significantly
@@ -158,10 +151,8 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
     const truncateStart = Date.now();
     await pool.query(`TRUNCATE test_data`);
 
-    const checkpoint2 = await context.getCheckpoint({ timeout: 20_000 });
+    const checksum2 = await context.getChecksums(['global[]'], { timeout: 20_000 });
     const truncateDuration = Date.now() - truncateStart;
-
-    const checksum2 = await context.storage!.getChecksums(checkpoint2, ['global[]']);
     const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
     expect(truncateCount).toEqual(numTransactions * perTransaction);
     const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
@@ -183,7 +174,7 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
     // 4. Another document to make sure the internal batching overflows
     //    to a second batch.
 
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     await context.updateSyncRules(`bucket_definitions:
   global:
     data:
@@ -226,10 +217,7 @@ function defineBatchTests(factory: storage.TestStorageFactory) {
     });
     await context.replicateSnapshot();
 
-    context.startStreaming();
-
-    const checkpoint = await context.getCheckpoint({ timeout: 50_000 });
-    const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+    const checksum = await context.getChecksums(['global[]'], { timeout: 50_000 });
     expect(checksum.get('global[]')!.count).toEqual((numDocs + 2) * 4);
   });
 

package/test/src/pg_test.test.ts

@@ -1,20 +1,20 @@
-import type { LookupFunction } from 'node:net';
+import { WalStream } from '@module/replication/WalStream.js';
+import { PostgresTypeResolver } from '@module/types/resolver.js';
 import * as dns from 'node:dns';
+import type { LookupFunction } from 'node:net';
 
 import * as pgwire from '@powersync/service-jpgwire';
 import {
   applyRowContext,
   CompatibilityContext,
-  SqliteInputRow,
+  CompatibilityEdition,
   DateTimeValue,
+  SqliteInputRow,
   TimeValue,
-  CompatibilityEdition,
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
 import { describe, expect, Mock, test, vi } from 'vitest';
 import { clearTestDb, connectPgPool, connectPgWire, TEST_CONNECTION_OPTIONS, TEST_URI } from './util.js';
-import { WalStream } from '@module/replication/WalStream.js';
-import { PostgresTypeResolver } from '@module/types/resolver.js';
 
 describe('connection options', () => {
   test('uses custom lookup', async () => {

package/test/src/resuming_snapshots.test.ts

@@ -1,27 +1,29 @@
-import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { describeWithStorage } from './util.js';
-import { WalStreamTestContext } from './wal_stream_utils.js';
-import { TestStorageFactory } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as timers from 'node:timers/promises';
-import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
+import { WalStreamTestContext } from './wal_stream_utils.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, function (factory) {
+  describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) {
     test('resuming initial replication (1)', async () => {
       // Stop early - likely to not include deleted row in first replication attempt.
-      await testResumingReplication(factory, 2000);
+      await testResumingReplication(factory, storageVersion, 2000);
     });
     test('resuming initial replication (2)', async () => {
       // Stop late - likely to include deleted row in first replication attempt.
-      await testResumingReplication(factory, 8000);
+      await testResumingReplication(factory, storageVersion, 8000);
     });
   });
 });
 
-async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
+async function testResumingReplication(
+  factory: StorageVersionTestContext['factory'],
+  storageVersion: number,
+  stopAfter: number
+) {
   // This tests interrupting and then resuming initial replication.
   // We interrupt replication after test_data1 has fully replicated, and
   // test_data2 has partially replicated.
@@ -33,7 +35,10 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   //    have been / have not been replicated at that point is not deterministic.
   //    We do allow for some variation in the test results to account for this.
 
-  await using context = await WalStreamTestContext.open(factory, { walStreamOptions: { snapshotChunkLength: 1000 } });
+  await using context = await WalStreamTestContext.open(factory, {
+    storageVersion,
+    walStreamOptions: { snapshotChunkLength: 1000 }
+  });
 
   await context.updateSyncRules(`bucket_definitions:
   global:
@@ -74,8 +79,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
       await context.dispose();
     })();
     // This confirms that initial replication was interrupted
-    const error = await p.catch((e) => e);
-    expect(error).toBeInstanceOf(ReplicationAbortedError);
+    await expect(p).rejects.toThrowError();
     done = true;
   } finally {
     done = true;
@@ -84,6 +88,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   // Bypass the usual "clear db on factory open" step.
   await using context2 = await WalStreamTestContext.open(factory, {
     doNotClear: true,
+    storageVersion,
     walStreamOptions: { snapshotChunkLength: 1000 }
   });
 
@@ -104,7 +109,6 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   await context2.loadNextSyncRules();
   await context2.replicateSnapshot();
 
-  context2.startStreaming();
   const data = await context2.getBucketData('global[]', undefined, {});
 
   const deletedRowOps = data.filter(
@@ -127,14 +131,14 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
     // so it's not in the resulting ops at all.
   }
 
-  expect(updatedRowOps.length).toEqual(2);
+  expect(updatedRowOps.length).toBeGreaterThanOrEqual(2);
   // description for the first op could be 'foo' or 'update1'.
   // We only test the final version.
-  expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
+  expect(JSON.parse(updatedRowOps[updatedRowOps.length - 1].data as string).description).toEqual('update1');
 
-  expect(insertedRowOps.length).toEqual(2);
+  expect(insertedRowOps.length).toBeGreaterThanOrEqual(1);
   expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
-  expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+  expect(JSON.parse(insertedRowOps[insertedRowOps.length - 1].data as string).description).toEqual('insert1');
 
   // 1000 of test_data1 during first replication attempt.
   // N >= 1000 of test_data2 during first replication attempt.
@@ -145,12 +149,12 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   // This adds 2 ops.
   // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
   // However, this is not deterministic.
-  const expectedCount = 11002 + deletedRowOps.length;
+  const expectedCount = 11000 - 2 + insertedRowOps.length + updatedRowOps.length + deletedRowOps.length;
   expect(data.length).toEqual(expectedCount);
 
   const replicatedCount =
     ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
 
   // With resumable replication, there should be no need to re-replicate anything.
-  expect(replicatedCount).toEqual(expectedCount);
+  expect(replicatedCount).toBeGreaterThanOrEqual(expectedCount);
 }

package/test/src/route_api_adapter.test.ts

@@ -1,7 +1,7 @@
-import { describe, expect, test } from 'vitest';
-import { clearTestDb, connectPgPool } from './util.js';
 import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
 import { TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { clearTestDb, connectPgPool } from './util.js';
 
 describe('PostgresRouteAPIAdapter tests', () => {
   test('infers connection schema', async () => {
@@ -20,7 +20,9 @@ describe('PostgresRouteAPIAdapter tests', () => {
       `);
 
       const schema = await api.getConnectionSchema();
-      expect(schema).toStrictEqual([
+      // Ignore any other potential schemas in the test database, for example the 'powersync' schema.
+      const filtered = schema.filter((s) => s.name == 'public');
+      expect(filtered).toStrictEqual([
         {
           name: 'public',
           tables: [