@powersync/service-module-postgres 0.16.1 → 0.16.3

package/test/src/pg_test.test.ts

@@ -1,4 +1,3 @@
-import { constructAfterRecord } from '@module/utils/pgwire_utils.js';
 import * as pgwire from '@powersync/service-jpgwire';
 import {
   applyRowContext,
@@ -11,6 +10,8 @@ import {
 import { describe, expect, test } from 'vitest';
 import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js';
 import { WalStream } from '@module/replication/WalStream.js';
+import { PostgresTypeResolver } from '@module/types/resolver.js';
+import { CustomTypeRegistry } from '@module/types/registry.js';
 
 describe('pg data types', () => {
   async function setupTable(db: pgwire.PgClient) {
@@ -382,7 +383,7 @@ VALUES(10, ARRAY['null']::TEXT[]);
         }
       });
 
-      const transformed = await getReplicationTx(replicationStream);
+      const transformed = await getReplicationTx(db, replicationStream);
       await pg.end();
 
       checkResults(transformed);
@@ -419,7 +420,7 @@ VALUES(10, ARRAY['null']::TEXT[]);
         }
       });
 
-      const transformed = await getReplicationTx(replicationStream);
+      const transformed = await getReplicationTx(db, replicationStream);
       await pg.end();
 
       checkResultArrays(transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)));
@@ -470,17 +471,163 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
       await db.end();
     }
   });
+
+  test('test replication - custom types', async () => {
+    const db = await connectPgPool();
+    try {
+      await clearTestDb(db);
+      await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`);
+      await db.query(`CREATE TYPE composite AS (foo rating_value[], bar TEXT);`);
+      await db.query(`CREATE TYPE nested_composite AS (a BOOLEAN, b composite);`);
+      await db.query(`CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')`);
+
+      await db.query(`CREATE TABLE test_custom(
+        id serial primary key,
+        rating rating_value,
+        composite composite,
+        nested_composite nested_composite,
+        boxes box[],
+        mood mood
+      );`);
+
+      const slotName = 'test_slot';
+
+      await db.query({
+        statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
+        params: [{ type: 'varchar', value: slotName }]
+      });
+
+      await db.query({
+        statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`,
+        params: [{ type: 'varchar', value: slotName }]
+      });
+
+      await db.query(`
+        INSERT INTO test_custom
+          (rating, composite, nested_composite, boxes, mood)
+        VALUES (
+          1,
+          (ARRAY[2,3], 'bar'),
+          (TRUE, (ARRAY[2,3], 'bar')),
+          ARRAY[box(point '(1,2)', point '(3,4)'), box(point '(5, 6)', point '(7,8)')],
+          'happy'
+        );
+      `);
+
+      const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI);
+      const replicationStream = await pg.logicalReplication({
+        slot: slotName,
+        options: {
+          proto_version: '1',
+          publication_names: 'powersync'
+        }
+      });
+
+      const [transformed] = await getReplicationTx(db, replicationStream);
+      await pg.end();
+
+      const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY);
+      expect(oldFormat).toMatchObject({
+        rating: '1',
+        composite: '("{2,3}",bar)',
+        nested_composite: '(t,"(""{2,3}"",bar)")',
+        boxes: '["(3","4)","(1","2);(7","8)","(5","6)"]',
+        mood: 'happy'
+      });
+
+      const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      expect(newFormat).toMatchObject({
+        rating: 1,
+        composite: '{"foo":[2.0,3.0],"bar":"bar"}',
+        nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}',
+        boxes: JSON.stringify(['(3,4),(1,2)', '(7,8),(5,6)']),
+        mood: 'happy'
+      });
+    } finally {
+      await db.end();
+    }
+  });
+
+  test('test replication - multiranges', async () => {
+    const db = await connectPgPool();
+
+    if (!(await new PostgresTypeResolver(new CustomTypeRegistry(), db).supportsMultiRanges())) {
+      // This test requires Postgres 14 or later.
+      return;
+    }
+
+    try {
+      await clearTestDb(db);
+
+      await db.query(`CREATE TABLE test_custom(
+        id serial primary key,
+        ranges int4multirange[]
+      );`);
+
+      const slotName = 'test_slot';
+
+      await db.query({
+        statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
+        params: [{ type: 'varchar', value: slotName }]
+      });
+
+      await db.query({
+        statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`,
+        params: [{ type: 'varchar', value: slotName }]
+      });
+
+      await db.query(`
+        INSERT INTO test_custom
+          (ranges)
+        VALUES (
+          ARRAY[int4multirange(int4range(2, 4), int4range(5, 7, '(]'))]::int4multirange[]
+        );
+      `);
+
+      const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI);
+      const replicationStream = await pg.logicalReplication({
+        slot: slotName,
+        options: {
+          proto_version: '1',
+          publication_names: 'powersync'
+        }
+      });
+
+      const [transformed] = await getReplicationTx(db, replicationStream);
+      await pg.end();
+
+      const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY);
+      expect(oldFormat).toMatchObject({
+        ranges: '{"{[2,4),[6,8)}"}'
+      });
+
+      const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      expect(newFormat).toMatchObject({
+        ranges: JSON.stringify([
+          [
+            { lower: 2, upper: 4, lower_exclusive: 0, upper_exclusive: 1 },
+            { lower: 6, upper: 8, lower_exclusive: 0, upper_exclusive: 1 }
+          ]
+        ])
+      });
+    } finally {
+      await db.end();
+    }
+  });
 });
 
 /**
  * Return all the inserts from the first transaction in the replication stream.
  */
-async function getReplicationTx(replicationStream: pgwire.ReplicationStream) {
+async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) {
+  const typeCache = new PostgresTypeResolver(new CustomTypeRegistry(), db);
+  await typeCache.fetchTypesForSchema();
+
   let transformed: SqliteInputRow[] = [];
   for await (const batch of replicationStream.pgoutputDecode()) {
     for (const msg of batch.messages) {
       if (msg.tag == 'insert') {
-        transformed.push(constructAfterRecord(msg));
+        transformed.push(typeCache.constructAfterRecord(msg));
       } else if (msg.tag == 'commit') {
         return transformed;
       }

package/test/src/route_api_adapter.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, test } from 'vitest';
+import { clearTestDb, connectPgPool } from './util.js';
+import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js';
+import { TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from '@powersync/service-sync-rules';
+
+describe('PostgresRouteAPIAdapter tests', () => {
+  test('infers connection schema', async () => {
+    const db = await connectPgPool();
+    try {
+      await clearTestDb(db);
+      const api = new PostgresRouteAPIAdapter(db);
+
+      await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5)`);
+      await db.query(`
+        CREATE TABLE test_users (
+            id TEXT NOT NULL PRIMARY KEY,
+            is_admin BOOLEAN,
+            rating RATING_VALUE
+        );
+      `);
+
+      const schema = await api.getConnectionSchema();
+      expect(schema).toStrictEqual([
+        {
+          name: 'public',
+          tables: [
+            {
+              name: 'test_users',
+              columns: [
+                {
+                  internal_type: 'text',
+                  name: 'id',
+                  pg_type: 'text',
+                  sqlite_type: TYPE_TEXT,
+                  type: 'text'
+                },
+                {
+                  internal_type: 'boolean',
+                  name: 'is_admin',
+                  pg_type: 'bool',
+                  sqlite_type: TYPE_INTEGER,
+                  type: 'boolean'
+                },
+                {
+                  internal_type: 'rating_value',
+                  name: 'rating',
+                  pg_type: 'rating_value',
+                  sqlite_type: TYPE_REAL,
+                  type: 'rating_value'
+                }
+              ]
+            }
+          ]
+        }
+      ]);
+    } finally {
+      await db.end();
+    }
+  });
+});

package/test/src/schema_changes.test.ts

@@ -590,4 +590,36 @@ function defineTests(factory: storage.TestStorageFactory) {
 
     expect(failures).toEqual([]);
   });
+
+  test('custom types', async () => {
+    await using context = await WalStreamTestContext.open(factory);
+
+    await context.updateSyncRules(`
+streams:
+  stream:
+    query: SELECT * FROM "test_data"
+
+config:
+  edition: 2
+`);
+
+    const { pool } = context;
+    await pool.query(`CREATE TABLE test_data(id text primary key);`);
+    await pool.query(`INSERT INTO test_data(id) VALUES ('t1')`);
+
+    await context.replicateSnapshot();
+    context.startStreaming();
+
+    await pool.query(
+      { statement: `CREATE TYPE composite AS (foo bool, bar int4);` },
+      { statement: `ALTER TABLE test_data ADD COLUMN other composite;` },
+      { statement: `UPDATE test_data SET other = ROW(TRUE, 2)::composite;` }
+    );
+
+    const data = await context.getBucketData('1#stream|0[]');
+    expect(data).toMatchObject([
+      putOp('test_data', { id: 't1' }),
+      putOp('test_data', { id: 't1', other: '{"foo":1,"bar":2}' })
+    ]);
+  });
 }

package/test/src/slow_tests.test.ts

@@ -19,6 +19,7 @@ import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import * as timers from 'node:timers/promises';
+import { CustomTypeRegistry } from '@module/types/registry.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('slow tests', function () {
   describeWithStorage({ timeout: 120_000 }, function (factory) {
@@ -68,7 +69,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
   });
 
   async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) {
-    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+    const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
     const replicationConnection = await connections.replicationConnection();
     const pool = connections.pool;
     await clearTestDb(pool);
@@ -329,7 +330,7 @@ bucket_definitions:
       await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
       i += 1;
 
-      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+      const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
       const replicationConnection = await connections.replicationConnection();
 
       abortController = new AbortController();

package/test/src/types/registry.test.ts

@@ -0,0 +1,149 @@
+import { describe, expect, test, beforeEach } from 'vitest';
+import { CustomTypeRegistry } from '@module/types/registry.js';
+import { CHAR_CODE_COMMA, PgTypeOid } from '@powersync/service-jpgwire';
+import {
+  applyValueContext,
+  CompatibilityContext,
+  CompatibilityEdition,
+  toSyncRulesValue
+} from '@powersync/service-sync-rules';
+
+describe('custom type registry', () => {
+  let registry: CustomTypeRegistry;
+
+  beforeEach(() => {
+    registry = new CustomTypeRegistry();
+  });
+
+  function checkResult(raw: string, type: number, old: any, fixed: any) {
+    const input = registry.decodeDatabaseValue(raw, type);
+    const syncRulesValue = toSyncRulesValue(input);
+
+    expect(applyValueContext(syncRulesValue, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toStrictEqual(old);
+    expect(
+      applyValueContext(syncRulesValue, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS))
+    ).toStrictEqual(fixed);
+  }
+
+  test('domain types', () => {
+    registry.setDomainType(1337, PgTypeOid.INT4); // create domain wrapping integer
+    checkResult('12', 1337, '12', 12n); // Should be raw text value without fix, parsed as inner type if enabled
+  });
+
+  test('array of domain types', () => {
+    registry.setDomainType(1337, PgTypeOid.INT4);
+    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
+
+    checkResult('{1,2,3}', 1338, '{1,2,3}', '[1,2,3]');
+  });
+
+  test('nested array through domain type', () => {
+    registry.setDomainType(1337, PgTypeOid.INT4);
+    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
+    registry.setDomainType(1339, 1338);
+
+    checkResult('{1,2,3}', 1339, '{1,2,3}', '[1,2,3]');
+
+    registry.set(1400, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1339, sqliteType: () => 'text' });
+    checkResult('{{1,2,3}}', 1400, '{{1,2,3}}', '[[1,2,3]]');
+  });
+
+  test('structure', () => {
+    // create type c1 AS (a bool, b integer, c text[]);
+    registry.set(1337, {
+      type: 'composite',
+      sqliteType: () => 'text',
+      members: [
+        { name: 'a', typeId: PgTypeOid.BOOL },
+        { name: 'b', typeId: PgTypeOid.INT4 },
+        { name: 'c', typeId: 1009 } // text array
+      ]
+    });
+
+    // SELECT (TRUE, 123, ARRAY['foo', 'bar'])::c1;
+    checkResult('(t,123,"{foo,bar}")', 1337, '(t,123,"{foo,bar}")', '{"a":1,"b":123,"c":["foo","bar"]}');
+  });
+
+  test('array of structure', () => {
+    // create type c1 AS (a bool, b integer, c text[]);
+    registry.set(1337, {
+      type: 'composite',
+      sqliteType: () => 'text',
+      members: [
+        { name: 'a', typeId: PgTypeOid.BOOL },
+        { name: 'b', typeId: PgTypeOid.INT4 },
+        { name: 'c', typeId: 1009 } // text array
+      ]
+    });
+    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
+
+    // SELECT ARRAY[(TRUE, 123, ARRAY['foo', 'bar']),(FALSE, NULL, ARRAY[]::text[])]::c1[];
+    checkResult(
+      '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
+      1338,
+      '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
+      '[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":null,"c":[]}]'
+    );
+  });
+
+  test('domain type of structure', () => {
+    registry.set(1337, {
+      type: 'composite',
+      sqliteType: () => 'text',
+      members: [
+        { name: 'a', typeId: PgTypeOid.BOOL },
+        { name: 'b', typeId: PgTypeOid.INT4 }
+      ]
+    });
+    registry.setDomainType(1338, 1337);
+
+    checkResult('(t,123)', 1337, '(t,123)', '{"a":1,"b":123}');
+  });
+
+  test('structure of another structure', () => {
+    // CREATE TYPE c2 AS (a BOOLEAN, b INTEGER);
+    registry.set(1337, {
+      type: 'composite',
+      sqliteType: () => 'text',
+      members: [
+        { name: 'a', typeId: PgTypeOid.BOOL },
+        { name: 'b', typeId: PgTypeOid.INT4 }
+      ]
+    });
+    registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
+    // CREATE TYPE c3 (c c2[]);
+    registry.set(1339, {
+      type: 'composite',
+      sqliteType: () => 'text',
+      members: [{ name: 'c', typeId: 1338 }]
+    });
+
+    // SELECT ROW(ARRAY[(FALSE,2)]::c2[])::c3;
+    checkResult('("{""(f,2)""}")', 1339, '("{""(f,2)""}")', '{"c":[{"a":0,"b":2}]}');
+  });
+
+  test('range', () => {
+    registry.set(1337, {
+      type: 'range',
+      sqliteType: () => 'text',
+      innerId: PgTypeOid.INT2
+    });
+
+    checkResult('[1,2]', 1337, '[1,2]', '{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":0}');
+  });
+
+  test('multirange', () => {
+    registry.set(1337, {
+      type: 'multirange',
+      sqliteType: () => 'text',
+      innerId: PgTypeOid.INT2
+    });
+
+    checkResult(
+      '{[1,2),[3,4)}',
+      1337,
+      '{[1,2),[3,4)}',
+      '[{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":1},{"lower":3,"upper":4,"lower_exclusive":0,"upper_exclusive":1}]'
+    );
+  });
+});

package/test/src/util.ts

@@ -59,6 +59,22 @@ export async function clearTestDb(db: pgwire.PgClient) {
       await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`);
     }
   }
+
+  const domainRows = pgwire.pgwireRows(
+    await db.query(`
+      SELECT typname,typtype
+      FROM pg_type t
+        LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace
+      WHERE n.nspname = 'public' AND typarray != 0
+    `)
+  );
+  for (let row of domainRows) {
+    if (row.typtype == 'd') {
+      await db.query(`DROP DOMAIN public.${lib_postgres.escapeIdentifier(row.typname)} CASCADE`);
+    } else {
+      await db.query(`DROP TYPE public.${lib_postgres.escapeIdentifier(row.typname)} CASCADE`);
+    }
+  }
 }
 
 export async function connectPgWire(type?: 'replication' | 'standard') {

package/test/src/wal_stream.test.ts

@@ -324,4 +324,28 @@ bucket_definitions:
       // creating a new replication slot.
     }
   });
+
+  test('custom types', async () => {
+    await using context = await WalStreamTestContext.open(factory);
+
+    await context.updateSyncRules(`
+streams:
+  stream:
+    query: SELECT id, * FROM "test_data"
+
+config:
+  edition: 2
+`);
+
+    const { pool } = context;
+    await pool.query(`DROP TABLE IF EXISTS test_data`);
+    await pool.query(`CREATE TYPE composite AS (foo bool, bar int4);`);
+    await pool.query(`CREATE TABLE test_data(id text primary key, description composite);`);
+
+    await context.initializeReplication();
+    await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`);
+
+    const data = await context.getBucketData('1#stream|0[]');
+    expect(data).toMatchObject([putOp('test_data', { id: 't1', description: '{"foo":1,"bar":2}' })]);
+  });
 }

package/test/src/wal_stream_utils.ts

@@ -12,6 +12,7 @@ import {
 import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import * as pgwire from '@powersync/service-jpgwire';
 import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
+import { CustomTypeRegistry } from '@module/types/registry.js';
 
 export class WalStreamTestContext implements AsyncDisposable {
   private _walStream?: WalStream;
@@ -32,7 +33,7 @@ export class WalStreamTestContext implements AsyncDisposable {
     options?: { doNotClear?: boolean; walStreamOptions?: Partial<WalStreamOptions> }
   ) {
     const f = await factory({ doNotClear: options?.doNotClear });
-    const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
+    const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() });
 
     if (!options?.doNotClear) {
       await clearTestDb(connectionManager.pool);