@powersync/service-module-postgres 0.3.0 → 0.5.0

package/test/src/setup.ts

@@ -1,9 +1,13 @@
 import { container } from '@powersync/lib-services-framework';
 import { test_utils } from '@powersync/service-core-tests';
-import { beforeAll } from 'vitest';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
   await test_utils.initMetrics();
 });
+
+beforeEach(async () => {
+  await test_utils.resetMetrics();
+});

package/test/src/slow_tests.test.ts

@@ -7,6 +7,7 @@ import {
   connectPgPool,
   getClientCheckpoint,
   INITIALIZED_MONGO_STORAGE_FACTORY,
+  INITIALIZED_POSTGRES_STORAGE_FACTORY,
   TEST_CONNECTION_OPTIONS
 } from './util.js';
 
@@ -17,9 +18,10 @@ import { PgManager } from '@module/replication/PgManager.js';
 import { storage } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
+import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import * as timers from 'node:timers/promises';
 
-describe('slow tests - mongodb', function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () {
   // These are slow, inconsistent tests.
   // Not run on every test run, but we do run on CI, or when manually debugging issues.
   if (env.CI || env.SLOW_TESTS) {
@@ -30,6 +32,17 @@ describe('slow tests - mongodb', function () {
   }
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('slow tests - postgres', function () {
+  // These are slow, inconsistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
 function defineSlowTests(factory: storage.TestStorageFactory) {
   let walStream: WalStream | undefined;
   let connections: PgManager | undefined;
@@ -79,7 +92,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
     const replicationConnection = await connections.replicationConnection();
     const pool = connections.pool;
     await clearTestDb(pool);
-    const f = (await factory()) as mongo_storage.storage.MongoBucketStorage;
+    await using f = await factory();
 
     const syncRuleContent = `
 bucket_definitions:
@@ -174,15 +187,50 @@ bucket_definitions:
           }
 
           const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint);
-          const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-            .filter((row) => row._id.o <= checkpoint)
-            .map(mongo_storage.storage.mapOpEntry);
-          await storage.compact({ maxOpId: checkpoint });
-          const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-            .filter((row) => row._id.o <= checkpoint)
-            .map(mongo_storage.storage.mapOpEntry);
-
-          test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          if (f instanceof mongo_storage.storage.MongoBucketStorage) {
+            const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+              .filter((row) => row._id.o <= checkpoint)
+              .map(mongo_storage.storage.mapOpEntry);
+            await storage.compact({ maxOpId: checkpoint });
+            const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+              .filter((row) => row._id.o <= checkpoint)
+              .map(mongo_storage.storage.mapOpEntry);
+
+            test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) {
+            const { db } = f;
+            const opsBefore = (
+              await db.sql`
+                SELECT
+                  *
+                FROM
+                  bucket_data
+                WHERE
+                  op_id <= ${{ type: 'int8', value: checkpoint }}
+                ORDER BY
+                  op_id ASC
+              `
+                .decoded(postgres_storage.models.BucketData)
+                .rows()
+            ).map(postgres_storage.utils.mapOpEntry);
+            await storage.compact({ maxOpId: checkpoint });
+            const opsAfter = (
+              await db.sql`
+                SELECT
+                  *
+                FROM
+                  bucket_data
+                WHERE
+                  op_id <= ${{ type: 'int8', value: checkpoint }}
+                ORDER BY
+                  op_id ASC
+              `
+                .decoded(postgres_storage.models.BucketData)
+                .rows()
+            ).map(postgres_storage.utils.mapOpEntry);
+
+            test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          }
         }
       };
 
@@ -196,26 +244,66 @@ bucket_definitions:
       // Wait for replication to finish
       let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
 
-      // Check that all inserts have been deleted again
-      const docs = await f.db.current_data.find().toArray();
-      const transformed = docs.map((doc) => {
-        return bson.deserialize(doc.data.buffer) as SqliteRow;
-      });
-      expect(transformed).toEqual([]);
-
-      // Check that each PUT has a REMOVE
-      const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
-
-      // All a single bucket in this test
-      const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op));
-      const reduced = test_utils.reduceBucket(bucket);
-      expect(reduced).toMatchObject([
-        {
-          op_id: '0',
-          op: 'CLEAR'
-        }
-        // Should contain no additional data
-      ]);
+      if (f instanceof mongo_storage.storage.MongoBucketStorage) {
+        // Check that all inserts have been deleted again
+        const docs = await f.db.current_data.find().toArray();
+        const transformed = docs.map((doc) => {
+          return bson.deserialize(doc.data.buffer) as SqliteRow;
+        });
+        expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+
+        // All a single bucket in this test
+        const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op));
+        const reduced = test_utils.reduceBucket(bucket);
+        expect(reduced).toMatchObject([
+          {
+            op_id: '0',
+            op: 'CLEAR'
+          }
+          // Should contain no additional data
+        ]);
+      } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) {
+        const { db } = f;
+        // Check that all inserts have been deleted again
+        const docs = await db.sql`
+          SELECT
+            *
+          FROM
+            current_data
+        `
+          .decoded(postgres_storage.models.CurrentData)
+          .rows();
+        const transformed = docs.map((doc) => {
+          return bson.deserialize(doc.data) as SqliteRow;
+        });
+        expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await db.sql`
+          SELECT
+            *
+          FROM
+            bucket_data
+          ORDER BY
+            op_id ASC
+        `
+          .decoded(postgres_storage.models.BucketData)
+          .rows();
+
+        // All a single bucket in this test
+        const bucket = ops.map((op) => postgres_storage.utils.mapOpEntry(op));
+        const reduced = test_utils.reduceBucket(bucket);
+        expect(reduced).toMatchObject([
+          {
+            op_id: '0',
+            op: 'CLEAR'
+          }
+          // Should contain no additional data
+        ]);
+      }
     }
 
     abortController.abort();
@@ -231,7 +319,7 @@ bucket_definitions:
     async () => {
       const pool = await connectPgPool();
       await clearTestDb(pool);
-      const f = await factory();
+      await using f = await factory();
 
       const syncRuleContent = `
 bucket_definitions:

package/test/src/util.ts

@@ -1,10 +1,11 @@
 import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
 import * as types from '@module/types/types.js';
-import * as pg_utils from '@module/utils/pgwire_utils.js';
+import * as lib_postgres from '@powersync/lib-service-postgres';
 import { logger } from '@powersync/lib-services-framework';
 import { BucketStorageFactory, OpId } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
+import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import { env } from './env.js';
 
 export const TEST_URI = env.PG_TEST_URL;
@@ -14,6 +15,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF
   isCI: env.CI
 });
 
+export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({
+  url: env.PG_STORAGE_TEST_URL
+});
+
 export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   type: 'postgresql',
   uri: TEST_URI,
@@ -40,7 +45,7 @@ export async function clearTestDb(db: pgwire.PgClient) {
   for (let row of tableRows) {
     const name = row.table_name;
     if (name.startsWith('test_')) {
-      await db.query(`DROP TABLE public.${pg_utils.escapeIdentifier(name)}`);
+      await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`);
     }
   }
 }

package/test/src/wal_stream.test.ts

@@ -4,7 +4,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import * as crypto from 'crypto';
 import { describe, expect, test } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { env } from './env.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
 const BASIC_SYNC_RULES = `
@@ -14,10 +15,14 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
 
-describe('wal stream - mongodb', { timeout: 20_000 }, function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('wal stream - mongodb', { timeout: 20_000 }, function () {
   defineWalStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('wal stream - postgres', { timeout: 20_000 }, function () {
+  defineWalStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+});
+
 function defineWalStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
     await using context = await WalStreamTestContext.open(factory);

package/test/src/wal_stream_utils.ts

@@ -46,6 +46,7 @@ export class WalStreamTestContext implements AsyncDisposable {
     await this.streamPromise;
     await this.connectionManager.destroy();
     this.storage?.[Symbol.dispose]();
+    await this.factory?.[Symbol.asyncDispose]();
   }
 
   get pool() {

package/tsconfig.json

@@ -26,6 +26,9 @@
     },
     {
       "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-postgres"
     }
   ]
 }