@powersync/service-module-postgres 0.7.1 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @powersync/service-module-postgres
 
+## 0.8.0
+
+### Minor Changes
+
+- 436eee6: Minor optimizations to new checkpoint calulations.
+
+### Patch Changes
+
+- Updated dependencies [436eee6]
+- Updated dependencies [15283d4]
+- Updated dependencies [88d4cb3]
+- Updated dependencies [f55e36a]
+  - @powersync/service-core@1.7.0
+  - @powersync/service-sync-rules@0.24.0
+  - @powersync/lib-services-framework@0.5.2
+  - @powersync/lib-service-postgres@0.3.2
+
 ## 0.7.1
 
 ### Patch Changes

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.7.1",
+  "version": "0.8.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -28,20 +28,20 @@
     "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^9.0.1",
-    "@powersync/lib-service-postgres": "0.3.1",
-    "@powersync/lib-services-framework": "0.5.1",
-    "@powersync/service-core": "0.18.1",
+    "@powersync/lib-service-postgres": "0.3.2",
+    "@powersync/lib-services-framework": "0.5.2",
+    "@powersync/service-core": "1.7.0",
     "@powersync/service-jpgwire": "0.19.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.23.4",
+    "@powersync/service-sync-rules": "0.24.0",
     "@powersync/service-types": "0.8.0"
   },
   "devDependencies": {
     "@types/semver": "^7.5.4",
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.4.1",
-    "@powersync/service-module-mongodb-storage": "0.5.1",
-    "@powersync/service-module-postgres-storage": "0.3.1"
+    "@powersync/service-core-tests": "0.5.0",
+    "@powersync/service-module-mongodb-storage": "0.6.0",
+    "@powersync/service-module-postgres-storage": "0.4.0"
   },
   "scripts": {
     "build": "tsc -b",

package/test/src/checkpoints.test.ts

@@ -3,6 +3,7 @@ import { checkpointUserId, createWriteCheckpoint } from '@powersync/service-core
 import { describe, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
+import { env } from './env.js';
 
 import timers from 'node:timers/promises';
 
@@ -11,8 +12,8 @@ const BASIC_SYNC_RULES = `bucket_definitions:
     data:
       - SELECT id, description, other FROM "test_data"`;
 
-describe('checkpoint tests', () => {
-  test('write checkpoints', { timeout: 30_000 }, async () => {
+describe.skipIf(!(env.CI || env.SLOW_TESTS))('checkpoint tests', () => {
+  test('write checkpoints', { timeout: 50_000 }, async () => {
     const factory = INITIALIZED_MONGO_STORAGE_FACTORY;
     await using context = await WalStreamTestContext.open(factory);
 
@@ -25,13 +26,14 @@ describe('checkpoint tests', () => {
     await context.replicateSnapshot();
 
     context.startStreaming();
+    const storage = context.storage!;
 
     const controller = new AbortController();
     try {
-      const stream = context.factory.watchWriteCheckpoint(
-        checkpointUserId('test_user', 'test_client'),
-        controller.signal
-      );
+      const stream = storage.watchWriteCheckpoint({
+        user_id: checkpointUserId('test_user', 'test_client'),
+        signal: controller.signal
+      });
 
       let lastWriteCheckpoint: bigint | null = null;
 
@@ -57,10 +59,12 @@ describe('checkpoint tests', () => {
 
         const start = Date.now();
         while (lastWriteCheckpoint == null || lastWriteCheckpoint < BigInt(cp.writeCheckpoint)) {
-          if (Date.now() - start > 2_000) {
-            throw new Error(`Timeout while waiting for checkpoint`);
+          if (Date.now() - start > 5_000) {
+            throw new Error(
+              `Timeout while waiting for checkpoint. last: ${lastWriteCheckpoint}, waiting for: ${cp.writeCheckpoint}`
+            );
           }
-          await timers.setTimeout(0, undefined, { signal: controller.signal });
+          await timers.setTimeout(5, undefined, { signal: controller.signal });
         }
       }
     } finally {

package/test/src/slow_tests.test.ts

@@ -71,21 +71,13 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
   // Past issues that this could reproduce intermittently:
   // * Skipping LSNs after a keepalive message
   // * Skipping LSNs when source transactions overlap
-  test(
-    'repeated replication - basic',
-    async () => {
-      await testRepeatedReplication({ compact: false, maxBatchSize: 50, numBatches: 5 });
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
-
-  test(
-    'repeated replication - compacted',
-    async () => {
-      await testRepeatedReplication({ compact: true, maxBatchSize: 100, numBatches: 2 });
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
+  test('repeated replication - basic', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
+    await testRepeatedReplication({ compact: false, maxBatchSize: 50, numBatches: 5 });
+  });
+
+  test('repeated replication - compacted', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
+    await testRepeatedReplication({ compact: true, maxBatchSize: 100, numBatches: 2 });
+  });
 
   async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
     const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
@@ -101,7 +93,7 @@ bucket_definitions:
       - SELECT * FROM "test_data"
 `;
     const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-    using storage = f.getInstance(syncRules);
+    const storage = f.getInstance(syncRules);
     abortController = new AbortController();
     const options: WalStreamOptions = {
       abort_signal: abortController.signal,
@@ -314,116 +306,112 @@ bucket_definitions:
   //
   // If the first LSN does not correctly match with the first replication transaction,
   // we may miss some updates.
-  test(
-    'repeated initial replication',
-    async () => {
-      const pool = await connectPgPool();
-      await clearTestDb(pool);
-      await using f = await factory();
-
-      const syncRuleContent = `
+  test('repeated initial replication', { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }, async () => {
+    const pool = await connectPgPool();
+    await clearTestDb(pool);
+    await using f = await factory();
+
+    const syncRuleContent = `
 bucket_definitions:
   global:
     data:
       - SELECT id, description FROM "test_data"
 `;
-      const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-      using storage = f.getInstance(syncRules);
+    const syncRules = await f.updateSyncRules({ content: syncRuleContent });
+    const storage = f.getInstance(syncRules);
 
-      // 1. Setup some base data that will be replicated in initial replication
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+    // 1. Setup some base data that will be replicated in initial replication
+    await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
 
-      let statements: pgwire.Statement[] = [];
+    let statements: pgwire.Statement[] = [];
 
-      const n = Math.floor(Math.random() * 200);
-      for (let i = 0; i < n; i++) {
-        statements.push({
-          statement: `INSERT INTO test_data(description) VALUES('test_init')`
-        });
-      }
-      await pool.query(...statements);
-
-      const start = Date.now();
-      let i = 0;
-
-      while (Date.now() - start < TEST_DURATION_MS) {
-        // 2. Each iteration starts with a clean slate
-        await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
-        i += 1;
-
-        const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-        const replicationConnection = await connections.replicationConnection();
-
-        abortController = new AbortController();
-        const options: WalStreamOptions = {
-          abort_signal: abortController.signal,
-          connections,
-          storage: storage
-        };
-        walStream = new WalStream(options);
-
-        await storage.clear();
-
-        // 3. Start initial replication, then streaming, but don't wait for any of this
-        let initialReplicationDone = false;
-        streamPromise = (async () => {
-          await walStream.initReplication(replicationConnection);
-          await storage.autoActivate();
+    const n = Math.floor(Math.random() * 200);
+    for (let i = 0; i < n; i++) {
+      statements.push({
+        statement: `INSERT INTO test_data(description) VALUES('test_init')`
+      });
+    }
+    await pool.query(...statements);
+
+    const start = Date.now();
+    let i = 0;
+
+    while (Date.now() - start < TEST_DURATION_MS) {
+      // 2. Each iteration starts with a clean slate
+      await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
+      i += 1;
+
+      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+      const replicationConnection = await connections.replicationConnection();
+
+      abortController = new AbortController();
+      const options: WalStreamOptions = {
+        abort_signal: abortController.signal,
+        connections,
+        storage: storage
+      };
+      walStream = new WalStream(options);
+
+      await storage.clear();
+
+      // 3. Start initial replication, then streaming, but don't wait for any of this
+      let initialReplicationDone = false;
+      streamPromise = (async () => {
+        await walStream.initReplication(replicationConnection);
+        await storage.autoActivate();
+        initialReplicationDone = true;
+        await walStream.streamChanges(replicationConnection);
+      })()
+        .catch((e) => {
           initialReplicationDone = true;
-          await walStream.streamChanges(replicationConnection);
-        })()
-          .catch((e) => {
-            initialReplicationDone = true;
-            throw e;
-          })
-          .then((v) => {
-            return v;
-          });
+          throw e;
+        })
+        .then((v) => {
+          return v;
+        });
 
-        // 4. While initial replication is still running, write more changes
-        while (!initialReplicationDone) {
-          let statements: pgwire.Statement[] = [];
-          const n = Math.floor(Math.random() * 10) + 1;
-          for (let i = 0; i < n; i++) {
-            const description = `test${i}`;
-            statements.push({
-              statement: `INSERT INTO test_data(description) VALUES('test1') returning id as test_id`,
-              params: [{ type: 'varchar', value: description }]
-            });
-          }
-          const results = await pool.query(...statements);
-          const ids = results.results.map((sub) => {
-            return sub.rows[0][0] as string;
-          });
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
-          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-            return {
-              statement: `DELETE FROM test_data WHERE id = $1`,
-              params: [{ type: 'uuid', value: id }]
-            };
+      // 4. While initial replication is still running, write more changes
+      while (!initialReplicationDone) {
+        let statements: pgwire.Statement[] = [];
+        const n = Math.floor(Math.random() * 10) + 1;
+        for (let i = 0; i < n; i++) {
+          const description = `test${i}`;
+          statements.push({
+            statement: `INSERT INTO test_data(description) VALUES('test1') returning id as test_id`,
+            params: [{ type: 'varchar', value: description }]
           });
-          await pool.query(...deleteStatements);
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
-        }
-
-        // 5. Once initial replication is done, wait for the streaming changes to complete syncing.
-        // getClientCheckpoint() effectively waits for the above replication to complete
-        // Race with streamingPromise to catch replication errors here.
-        let checkpoint = await Promise.race([
-          getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
-          streamPromise
-        ]);
-        if (typeof checkpoint == undefined) {
-          // This indicates an issue with the test setup - streamingPromise completed instead
-          // of getClientCheckpoint()
-          throw new Error('Test failure - streamingPromise completed');
         }
+        const results = await pool.query(...statements);
+        const ids = results.results.map((sub) => {
+          return sub.rows[0][0] as string;
+        });
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
+        const deleteStatements: pgwire.Statement[] = ids.map((id) => {
+          return {
+            statement: `DELETE FROM test_data WHERE id = $1`,
+            params: [{ type: 'uuid', value: id }]
+          };
+        });
+        await pool.query(...deleteStatements);
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
+      }
 
-        abortController.abort();
-        await streamPromise;
-        await connections.end();
+      // 5. Once initial replication is done, wait for the streaming changes to complete syncing.
+      // getClientCheckpoint() effectively waits for the above replication to complete
+      // Race with streamingPromise to catch replication errors here.
+      let checkpoint = await Promise.race([
+        getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
+        streamPromise
+      ]);
+      if (typeof checkpoint == undefined) {
+        // This indicates an issue with the test setup - streamingPromise completed instead
+        // of getClientCheckpoint()
+        throw new Error('Test failure - streamingPromise completed');
       }
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
+
+      abortController.abort();
+      await streamPromise;
+      await connections.end();
+    }
+  });
 }

package/test/src/util.ts

@@ -62,7 +62,7 @@ export function connectPgPool() {
 
 export async function getClientCheckpoint(
   db: pgwire.PgClient,
-  bucketStorage: BucketStorageFactory,
+  storageFactory: BucketStorageFactory,
   options?: { timeout?: number }
 ): Promise<OpId> {
   const start = Date.now();
@@ -77,8 +77,9 @@ export async function getClientCheckpoint(
 
   logger.info(`Waiting for LSN checkpoint: ${lsn}`);
   while (Date.now() - start < timeout) {
-    const cp = await bucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
+    const storage = await storageFactory.getActiveStorage();
+    const cp = await storage?.getCheckpoint();
+    if (cp == null) {
       throw new Error('No sync rules available');
     }
     if (cp.lsn && cp.lsn >= lsn) {

package/test/src/wal_stream_utils.ts

@@ -45,7 +45,6 @@ export class WalStreamTestContext implements AsyncDisposable {
     this.abortController.abort();
     await this.streamPromise;
     await this.connectionManager.destroy();
-    this.storage?.[Symbol.dispose]();
     await this.factory?.[Symbol.asyncDispose]();
   }