@powersync/service-module-postgres 0.18.0 → 0.19.1

package/test/src/large_batch.test.ts

@@ -3,9 +3,17 @@ import { populateData } from '../../dist/utils/populate_test_data.js';
 import { env } from './env.js';
 import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
+import { CURRENT_STORAGE_VERSION } from '@powersync/service-core';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, defineBatchTests);
+  describeWithStorage(
+    {
+      timeout: 240_000,
+      // These tests are slow, so only test the current storage version
+      storageVersions: [CURRENT_STORAGE_VERSION]
+    },
+    defineBatchTests
+  );
 });
 
 const BASIC_SYNC_RULES = `bucket_definitions:
@@ -40,9 +48,7 @@ function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext
 
     const start = Date.now();
 
-    context.startStreaming();
-
-    const checksum = await context.getChecksums(['global[]'], { timeout: 50_000 });
+    const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
     const duration = Date.now() - start;
     const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
     expect(checksum.get('global[]')!.count).toEqual(operation_count);
@@ -87,7 +93,6 @@ function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext
       const start = Date.now();
 
       await context.replicateSnapshot();
-      context.startStreaming();
 
       const checksum = await context.getChecksums(['global[]'], { timeout: 100_000 });
       const duration = Date.now() - start;
@@ -138,8 +143,6 @@ function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext
 
     const start = Date.now();
 
-    context.startStreaming();
-
     const checksum = await context.getChecksums(['global[]']);
     const duration = Date.now() - start;
     const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
@@ -222,7 +225,6 @@ function defineBatchTests({ factory, storageVersion }: StorageVersionTestContext
     });
     await context.replicateSnapshot();
 
-    context.startStreaming();
     const checksum = await context.getChecksums(['global[]'], { timeout: 50_000 });
     expect(checksum.get('global[]')!.count).toEqual((numDocs + 2) * 4);
   });

package/test/src/pg_test.test.ts

@@ -1,20 +1,20 @@
-import type { LookupFunction } from 'node:net';
+import { WalStream } from '@module/replication/WalStream.js';
+import { PostgresTypeResolver } from '@module/types/resolver.js';
 import * as dns from 'node:dns';
+import type { LookupFunction } from 'node:net';
 
 import * as pgwire from '@powersync/service-jpgwire';
 import {
   applyRowContext,
   CompatibilityContext,
-  SqliteInputRow,
+  CompatibilityEdition,
   DateTimeValue,
+  SqliteInputRow,
   TimeValue,
-  CompatibilityEdition,
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
 import { describe, expect, Mock, test, vi } from 'vitest';
 import { clearTestDb, connectPgPool, connectPgWire, TEST_CONNECTION_OPTIONS, TEST_URI } from './util.js';
-import { WalStream } from '@module/replication/WalStream.js';
-import { PostgresTypeResolver } from '@module/types/resolver.js';
 
 describe('connection options', () => {
   test('uses custom lookup', async () => {

package/test/src/resuming_snapshots.test.ts

@@ -1,11 +1,10 @@
+import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { ReplicationMetric } from '@powersync/service-types';
+import * as timers from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 import { env } from './env.js';
 import { describeWithStorage, StorageVersionTestContext } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
-import { METRICS_HELPER } from '@powersync/service-core-tests';
-import { ReplicationMetric } from '@powersync/service-types';
-import * as timers from 'node:timers/promises';
-import { ReplicationAbortedError } from '@powersync/lib-services-framework';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
   describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) {
@@ -80,8 +79,7 @@ async function testResumingReplication(
       await context.dispose();
     })();
     // This confirms that initial replication was interrupted
-    const error = await p.catch((e) => e);
-    expect(error).toBeInstanceOf(ReplicationAbortedError);
+    await expect(p).rejects.toThrowError();
     done = true;
   } finally {
     done = true;
@@ -111,7 +109,6 @@ async function testResumingReplication(
   await context2.loadNextSyncRules();
   await context2.replicateSnapshot();
 
-  context2.startStreaming();
   const data = await context2.getBucketData('global[]', undefined, {});
 
   const deletedRowOps = data.filter(
@@ -134,14 +131,14 @@ async function testResumingReplication(
     // so it's not in the resulting ops at all.
   }
 
-  expect(updatedRowOps.length).toEqual(2);
+  expect(updatedRowOps.length).toBeGreaterThanOrEqual(2);
   // description for the first op could be 'foo' or 'update1'.
   // We only test the final version.
-  expect(JSON.parse(updatedRowOps[1].data as string).description).toEqual('update1');
+  expect(JSON.parse(updatedRowOps[updatedRowOps.length - 1].data as string).description).toEqual('update1');
 
-  expect(insertedRowOps.length).toEqual(2);
+  expect(insertedRowOps.length).toBeGreaterThanOrEqual(1);
   expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1');
-  expect(JSON.parse(insertedRowOps[1].data as string).description).toEqual('insert1');
+  expect(JSON.parse(insertedRowOps[insertedRowOps.length - 1].data as string).description).toEqual('insert1');
 
   // 1000 of test_data1 during first replication attempt.
   // N >= 1000 of test_data2 during first replication attempt.
@@ -152,12 +149,12 @@ async function testResumingReplication(
   // This adds 2 ops.
   // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000.
   // However, this is not deterministic.
-  const expectedCount = 11002 + deletedRowOps.length;
+  const expectedCount = 11000 - 2 + insertedRowOps.length + updatedRowOps.length + deletedRowOps.length;
   expect(data.length).toEqual(expectedCount);
 
   const replicatedCount =
     ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
 
   // With resumable replication, there should be no need to re-replicate anything.
-  expect(replicatedCount).toEqual(expectedCount);
+  expect(replicatedCount).toBeGreaterThanOrEqual(expectedCount);
 }

package/test/src/route_api_adapter.test.ts

@@ -1,7 +1,7 @@
-import { describe, expect, test } from 'vitest';
-import { clearTestDb, connectPgPool } from './util.js';
 import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
 import { TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { clearTestDb, connectPgPool } from './util.js';
 
 describe('PostgresRouteAPIAdapter tests', () => {
   test('infers connection schema', async () => {

package/test/src/schema_changes.test.ts

@@ -39,7 +39,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
@@ -62,13 +61,17 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     // Truncate - order doesn't matter
     expect(data.slice(2, 4).sort(compareIds)).toMatchObject([REMOVE_T1, REMOVE_T2]);
 
-    expect(data.slice(4)).toMatchObject([
-      // Snapshot insert
-      PUT_T3,
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
+    expect(data.slice(4, 5)).toMatchObject([
+      // Snapshot and/or replication insert
       PUT_T3
     ]);
+
+    if (data.length > 5) {
+      expect(data.slice(5)).toMatchObject([
+        // Replicated insert (optional duplication)
+        PUT_T3
+      ]);
+    }
   });
 
   test('add table', async () => {
@@ -78,7 +81,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     const { pool } = context;
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`CREATE TABLE test_data(id text primary key, description text)`);
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
@@ -86,17 +88,10 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     const data = await context.getBucketData('global[]');
 
     // "Reduce" the bucket to get a stable output to test.
+    // The specific operation sequence may vary depending on storage implementation, so just check the end result.
     // slice(1) to skip the CLEAR op.
     const reduced = reduceBucket(data).slice(1);
     expect(reduced.sort(compareIds)).toMatchObject([PUT_T1]);
-
-    expect(data).toMatchObject([
-      // Snapshot insert
-      PUT_T1,
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      PUT_T1
-    ]);
   });
 
   test('rename table (1)', async () => {
@@ -110,7 +105,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data_old(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data_old RENAME TO test_data` },
@@ -130,11 +124,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
       PUT_T1,
       PUT_T2
     ]);
-    expect(data.slice(2)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      PUT_T2
-    ]);
+    if (data.length > 2) {
+      expect(data.slice(2)).toMatchObject([
+        // Replicated insert
+        // May be de-duplicated
+        PUT_T2
+      ]);
+    }
   });
 
   test('rename table (2)', async () => {
@@ -153,7 +149,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data1(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data1 RENAME TO test_data2` },
@@ -183,11 +178,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
       putOp('test_data2', { id: 't1', description: 'test1' }),
       putOp('test_data2', { id: 't2', description: 'test2' })
     ]);
-    expect(data.slice(4)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      putOp('test_data2', { id: 't2', description: 'test2' })
-    ]);
+    if (data.length > 4) {
+      expect(data.slice(4)).toMatchObject([
+        // Replicated insert
+        // This may be de-duplicated
+        putOp('test_data2', { id: 't2', description: 'test2' })
+      ]);
+    }
   });
 
   test('rename table (3)', async () => {
@@ -202,7 +199,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data RENAME TO test_data_na` },
@@ -237,7 +233,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data REPLICA IDENTITY FULL` },
@@ -262,11 +257,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     // Snapshot - order doesn't matter
     expect(data.slice(2, 4).sort(compareIds)).toMatchObject([PUT_T1, PUT_T2]);
 
-    expect(data.slice(4).sort(compareIds)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      PUT_T2
-    ]);
+    if (data.length > 4) {
+      expect(data.slice(4).sort(compareIds)).toMatchObject([
+        // Replicated insert
+        // This may be de-duplicated
+        PUT_T2
+      ]);
+    }
   });
 
   test('change full replica id by adding column', async () => {
@@ -283,7 +280,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data ADD COLUMN other TEXT` },
@@ -305,11 +301,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
       putOp('test_data', { id: 't2', description: 'test2', other: null })
     ]);
 
-    expect(data.slice(4).sort(compareIds)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      putOp('test_data', { id: 't2', description: 'test2', other: null })
-    ]);
+    if (data.length > 4) {
+      expect(data.slice(4).sort(compareIds)).toMatchObject([
+        // Replicated insert
+        // This may be de-duplicated
+        putOp('test_data', { id: 't2', description: 'test2', other: null })
+      ]);
+    }
   });
 
   test('change default replica id by changing column type', async () => {
@@ -323,7 +321,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data ALTER COLUMN id TYPE varchar` },
@@ -342,11 +339,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     // Snapshot - order doesn't matter
     expect(data.slice(2, 4).sort(compareIds)).toMatchObject([PUT_T1, PUT_T2]);
 
-    expect(data.slice(4).sort(compareIds)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      PUT_T2
-    ]);
+    if (data.length > 4) {
+      expect(data.slice(4).sort(compareIds)).toMatchObject([
+        // Replicated insert
+        // May be de-duplicated
+        PUT_T2
+      ]);
+    }
   });
 
   test('change index id by changing column type', async () => {
@@ -365,7 +364,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
@@ -388,21 +386,7 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     const reduced = reduceBucket(data).slice(1);
     expect(reduced.sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]);
 
-    // Previously had more specific tests, but this varies too much based on timing:
-    // expect(data.slice(2, 4).sort(compareIds)).toMatchObject([
-    //   // Truncate - any order
-    //   REMOVE_T1,
-    //   REMOVE_T2
-    // ]);
-
-    // // Snapshot - order doesn't matter
-    // expect(data.slice(4, 7).sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]);
-
-    // expect(data.slice(7).sort(compareIds)).toMatchObject([
-    //   // Replicated insert
-    //   // We may eventually be able to de-duplicate this
-    //   PUT_T3
-    // ]);
+    // Previously had more specific tests, but this varies too much based on timing.
   });
 
   test('add to publication', async () => {
@@ -420,7 +404,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
@@ -436,11 +419,13 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
       PUT_T3
     ]);
 
-    expect(data.slice(3)).toMatchObject([
-      // Replicated insert
-      // We may eventually be able to de-duplicate this
-      PUT_T3
-    ]);
+    if (data.length > 3) {
+      expect(data.slice(3)).toMatchObject([
+        // Replicated insert
+        // May be de-duplicated
+        PUT_T3
+      ]);
+    }
 
     // "Reduce" the bucket to get a stable output to test.
     // slice(1) to skip the CLEAR op.
@@ -464,7 +449,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_other(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_other(id, description) VALUES('t2', 'test2')`);
 
@@ -489,7 +473,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
@@ -532,7 +515,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t1', 'test1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(`INSERT INTO test_data(id, description) VALUES('t2', 'test2')`);
 
@@ -586,7 +568,6 @@ function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
     await pool.query(`INSERT INTO test_data_old(id, num) VALUES('t2', 0)`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `ALTER TABLE test_data_old RENAME TO test_data` },
@@ -658,7 +639,6 @@ config:
     await pool.query(`INSERT INTO test_data(id) VALUES ('t1')`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
 
     await pool.query(
       { statement: `CREATE TYPE composite AS (foo bool, bar int4);` },
@@ -666,7 +646,7 @@ config:
       { statement: `UPDATE test_data SET other = ROW(TRUE, 2)::composite;` }
     );
 
-    const data = await context.getBucketData('1#stream|0[]');
+    const data = await context.getBucketData('stream|0[]');
     expect(data).toMatchObject([
       putOp('test_data', { id: 't1' }),
       putOp('test_data', { id: 't1', other: '{"foo":1,"bar":2}' })