@powersync/service-module-postgres 0.3.0 → 0.4.0

package/src/utils/pgwire_utils.ts

@@ -1,9 +1,7 @@
 // Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
 
 import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteJsonValue, SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
-
-import { logger } from '@powersync/lib-services-framework';
+import { SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
 
 /**
  * pgwire message -> SQLite row.
@@ -28,46 +26,3 @@ export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.Pg
   const record = pgwire.decodeTuple(message.relation, rawData);
   return toSyncRulesRow(record);
 }
-
-export function escapeIdentifier(identifier: string) {
-  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
-}
-
-export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam {
-  if (arg == null) {
-    return { type: 'varchar', value: null };
-  } else if (typeof arg == 'string') {
-    return { type: 'varchar', value: arg };
-  } else if (typeof arg == 'number') {
-    if (Number.isInteger(arg)) {
-      return { type: 'int8', value: arg };
-    } else {
-      return { type: 'float8', value: arg };
-    }
-  } else if (typeof arg == 'boolean') {
-    return { type: 'bool', value: arg };
-  } else if (typeof arg == 'bigint') {
-    return { type: 'int8', value: arg };
-  } else {
-    throw new Error(`Unsupported query parameter: ${typeof arg}`);
-  }
-}
-
-export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
-export async function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
-
-/**
- * Retry a simple query - up to 2 attempts total.
- */
-export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
-  for (let tries = 2; ; tries--) {
-    try {
-      return await db.query(...args);
-    } catch (e) {
-      if (tries == 1) {
-        throw e;
-      }
-      logger.warn('Query error, retrying', e);
-    }
-  }
-}

package/test/src/__snapshots__/schema_changes.test.ts.snap

@@ -0,0 +1,5 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`schema changes - mongodb > add to publication (not in sync rules) 1`] = `0`;
+
+exports[`schema changes - postgres > add to publication (not in sync rules) 1`] = `16384`;

package/test/src/env.ts

@@ -2,7 +2,10 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
+  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'),
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false'),
-  SLOW_TESTS: utils.type.boolean.default('false')
+  SLOW_TESTS: utils.type.boolean.default('false'),
+  TEST_MONGO_STORAGE: utils.type.boolean.default('true'),
+  TEST_POSTGRES_STORAGE: utils.type.boolean.default('true')
 });

package/test/src/large_batch.test.ts

@@ -3,10 +3,14 @@ import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 import { populateData } from '../../dist/utils/populate_test_data.js';
 import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from './util.js';
+import {
+  INITIALIZED_MONGO_STORAGE_FACTORY,
+  INITIALIZED_POSTGRES_STORAGE_FACTORY,
+  TEST_CONNECTION_OPTIONS
+} from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
-describe('batch replication tests - mongodb', { timeout: 120_000 }, function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { timeout: 120_000 }, function () {
   // These are slow but consistent tests.
   // Not run on every test run, but we do run on CI, or when manually debugging issues.
   if (env.CI || env.SLOW_TESTS) {
@@ -17,6 +21,17 @@ describe('batch replication tests - mongodb', { timeout: 120_000 }, function ()
   }
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 240_000 }, function () {
+  // These are slow but consistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineBatchTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
 const BASIC_SYNC_RULES = `bucket_definitions:
   global:
     data:

package/test/src/schema_changes.test.ts

@@ -3,13 +3,18 @@ import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 
 import { storage } from '@powersync/service-core';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { env } from './env.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
-describe('schema changes', { timeout: 20_000 }, function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('schema changes - mongodb', { timeout: 20_000 }, function () {
   defineTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('schema changes - postgres', { timeout: 20_000 }, function () {
+  defineTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+});
+
 const BASIC_SYNC_RULES = `
 bucket_definitions:
   global:
@@ -432,7 +437,7 @@ function defineTests(factory: storage.TestStorageFactory) {
     expect(data).toMatchObject([]);
 
     const metrics = await storage.factory.getStorageMetrics();
-    expect(metrics.replication_size_bytes).toEqual(0);
+    expect(metrics.replication_size_bytes).toMatchSnapshot();
   });
 
   test('replica identity nothing', async () => {

package/test/src/setup.ts

@@ -1,9 +1,13 @@
 import { container } from '@powersync/lib-services-framework';
 import { test_utils } from '@powersync/service-core-tests';
-import { beforeAll } from 'vitest';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
   await test_utils.initMetrics();
 });
+
+beforeEach(async () => {
+  await test_utils.resetMetrics();
+});

package/test/src/slow_tests.test.ts

@@ -7,6 +7,7 @@ import {
   connectPgPool,
   getClientCheckpoint,
   INITIALIZED_MONGO_STORAGE_FACTORY,
+  INITIALIZED_POSTGRES_STORAGE_FACTORY,
   TEST_CONNECTION_OPTIONS
 } from './util.js';
 
@@ -17,9 +18,10 @@ import { PgManager } from '@module/replication/PgManager.js';
 import { storage } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
+import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import * as timers from 'node:timers/promises';
 
-describe('slow tests - mongodb', function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () {
   // These are slow, inconsistent tests.
   // Not run on every test run, but we do run on CI, or when manually debugging issues.
   if (env.CI || env.SLOW_TESTS) {
@@ -30,6 +32,17 @@ describe('slow tests - mongodb', function () {
   }
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('slow tests - postgres', function () {
+  // These are slow, inconsistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
 function defineSlowTests(factory: storage.TestStorageFactory) {
   let walStream: WalStream | undefined;
   let connections: PgManager | undefined;
@@ -79,7 +92,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
     const replicationConnection = await connections.replicationConnection();
     const pool = connections.pool;
     await clearTestDb(pool);
-    const f = (await factory()) as mongo_storage.storage.MongoBucketStorage;
+    await using f = await factory();
 
     const syncRuleContent = `
 bucket_definitions:
@@ -174,15 +187,50 @@ bucket_definitions:
           }
 
           const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint);
-          const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-            .filter((row) => row._id.o <= checkpoint)
-            .map(mongo_storage.storage.mapOpEntry);
-          await storage.compact({ maxOpId: checkpoint });
-          const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
-            .filter((row) => row._id.o <= checkpoint)
-            .map(mongo_storage.storage.mapOpEntry);
-
-          test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          if (f instanceof mongo_storage.storage.MongoBucketStorage) {
+            const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+              .filter((row) => row._id.o <= checkpoint)
+              .map(mongo_storage.storage.mapOpEntry);
+            await storage.compact({ maxOpId: checkpoint });
+            const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
+              .filter((row) => row._id.o <= checkpoint)
+              .map(mongo_storage.storage.mapOpEntry);
+
+            test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) {
+            const { db } = f;
+            const opsBefore = (
+              await db.sql`
+                SELECT
+                  *
+                FROM
+                  bucket_data
+                WHERE
+                  op_id <= ${{ type: 'int8', value: checkpoint }}
+                ORDER BY
+                  op_id ASC
+              `
+                .decoded(postgres_storage.models.BucketData)
+                .rows()
+            ).map(postgres_storage.utils.mapOpEntry);
+            await storage.compact({ maxOpId: checkpoint });
+            const opsAfter = (
+              await db.sql`
+                SELECT
+                  *
+                FROM
+                  bucket_data
+                WHERE
+                  op_id <= ${{ type: 'int8', value: checkpoint }}
+                ORDER BY
+                  op_id ASC
+              `
+                .decoded(postgres_storage.models.BucketData)
+                .rows()
+            ).map(postgres_storage.utils.mapOpEntry);
+
+            test_utils.validateCompactedBucket(opsBefore, opsAfter);
+          }
         }
       };
 
@@ -196,26 +244,66 @@ bucket_definitions:
       // Wait for replication to finish
       let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
 
-      // Check that all inserts have been deleted again
-      const docs = await f.db.current_data.find().toArray();
-      const transformed = docs.map((doc) => {
-        return bson.deserialize(doc.data.buffer) as SqliteRow;
-      });
-      expect(transformed).toEqual([]);
-
-      // Check that each PUT has a REMOVE
-      const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
-
-      // All a single bucket in this test
-      const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op));
-      const reduced = test_utils.reduceBucket(bucket);
-      expect(reduced).toMatchObject([
-        {
-          op_id: '0',
-          op: 'CLEAR'
-        }
-        // Should contain no additional data
-      ]);
+      if (f instanceof mongo_storage.storage.MongoBucketStorage) {
+        // Check that all inserts have been deleted again
+        const docs = await f.db.current_data.find().toArray();
+        const transformed = docs.map((doc) => {
+          return bson.deserialize(doc.data.buffer) as SqliteRow;
+        });
+        expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+
+        // All a single bucket in this test
+        const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op));
+        const reduced = test_utils.reduceBucket(bucket);
+        expect(reduced).toMatchObject([
+          {
+            op_id: '0',
+            op: 'CLEAR'
+          }
+          // Should contain no additional data
+        ]);
+      } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) {
+        const { db } = f;
+        // Check that all inserts have been deleted again
+        const docs = await db.sql`
+          SELECT
+            *
+          FROM
+            current_data
+        `
+          .decoded(postgres_storage.models.CurrentData)
+          .rows();
+        const transformed = docs.map((doc) => {
+          return bson.deserialize(doc.data) as SqliteRow;
+        });
+        expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await db.sql`
+          SELECT
+            *
+          FROM
+            bucket_data
+          ORDER BY
+            op_id ASC
+        `
+          .decoded(postgres_storage.models.BucketData)
+          .rows();
+
+        // All a single bucket in this test
+        const bucket = ops.map((op) => postgres_storage.utils.mapOpEntry(op));
+        const reduced = test_utils.reduceBucket(bucket);
+        expect(reduced).toMatchObject([
+          {
+            op_id: '0',
+            op: 'CLEAR'
+          }
+          // Should contain no additional data
+        ]);
+      }
     }
 
     abortController.abort();
@@ -231,7 +319,7 @@ bucket_definitions:
     async () => {
       const pool = await connectPgPool();
       await clearTestDb(pool);
-      const f = await factory();
+      await using f = await factory();
 
       const syncRuleContent = `
 bucket_definitions:

package/test/src/util.ts

@@ -1,10 +1,11 @@
 import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
 import * as types from '@module/types/types.js';
-import * as pg_utils from '@module/utils/pgwire_utils.js';
+import * as lib_postgres from '@powersync/lib-service-postgres';
 import { logger } from '@powersync/lib-services-framework';
 import { BucketStorageFactory, OpId } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
+import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import { env } from './env.js';
 
 export const TEST_URI = env.PG_TEST_URL;
@@ -14,6 +15,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF
   isCI: env.CI
 });
 
+export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({
+  url: env.PG_STORAGE_TEST_URL
+});
+
 export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   type: 'postgresql',
   uri: TEST_URI,
@@ -40,7 +45,7 @@ export async function clearTestDb(db: pgwire.PgClient) {
   for (let row of tableRows) {
     const name = row.table_name;
     if (name.startsWith('test_')) {
-      await db.query(`DROP TABLE public.${pg_utils.escapeIdentifier(name)}`);
+      await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`);
     }
   }
 }

package/test/src/wal_stream.test.ts

@@ -4,7 +4,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import * as crypto from 'crypto';
 import { describe, expect, test } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { env } from './env.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
 
 const BASIC_SYNC_RULES = `
@@ -14,10 +15,14 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
 
-describe('wal stream - mongodb', { timeout: 20_000 }, function () {
+describe.skipIf(!env.TEST_MONGO_STORAGE)('wal stream - mongodb', { timeout: 20_000 }, function () {
   defineWalStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 });
 
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('wal stream - postgres', { timeout: 20_000 }, function () {
+  defineWalStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+});
+
 function defineWalStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
     await using context = await WalStreamTestContext.open(factory);

package/test/src/wal_stream_utils.ts

@@ -46,6 +46,7 @@ export class WalStreamTestContext implements AsyncDisposable {
     await this.streamPromise;
     await this.connectionManager.destroy();
     this.storage?.[Symbol.dispose]();
+    await this.factory?.[Symbol.asyncDispose]();
   }
 
   get pool() {

package/tsconfig.json

@@ -26,6 +26,9 @@
     },
     {
       "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-postgres"
     }
   ]
 }