@powersync/service-module-postgres 0.0.0-dev-20240918092408

package/src/utils/migration_lib.ts

@@ -0,0 +1,79 @@
+import * as pgwire from '@powersync/service-jpgwire';
+
+export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
+
+interface Migration {
+  id: number;
+  name: string;
+  up: MigrationFunction;
+}
+
+// Very loosely based on https://github.com/porsager/postgres-shift/
+export class Migrations {
+  private migrations: Migration[] = [];
+
+  add(id: number, name: string, up: MigrationFunction) {
+    if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
+      throw new Error('Migration ids must be strictly incrementing');
+    }
+    this.migrations.push({ id, up, name });
+  }
+
+  async up(db: pgwire.PgConnection) {
+    await db.query('BEGIN');
+    try {
+      await this.ensureMigrationsTable(db);
+      const current = await this.getCurrentMigration(db);
+      let currentId = current ? current.id : 0;
+
+      for (let migration of this.migrations) {
+        if (migration.id <= currentId) {
+          continue;
+        }
+        await migration.up(db);
+
+        await db.query({
+          statement: `
+      insert into migrations (
+        migration_id,
+        name
+      ) values (
+        $1,
+        $2
+      )
+    `,
+          params: [
+            { type: 'int4', value: migration.id },
+            { type: 'varchar', value: migration.name }
+          ]
+        });
+      }
+
+      await db.query('COMMIT');
+    } catch (e) {
+      await db.query('ROLLBACK');
+      throw e;
+    }
+  }
+
+  getCurrentMigration(db: pgwire.PgConnection) {
+    return db
+      .query(
+        `
+      select migration_id as id from migrations
+      order by migration_id desc
+      limit 1
+    `
+      )
+      .then((results) => ({ id: results.rows[0][0] as number }));
+  }
+
+  async ensureMigrationsTable(db: pgwire.PgConnection) {
+    await db.query(`create table if not exists migrations (
+        migration_id serial primary key,
+        created_at timestamp with time zone not null default now(),
+        name text
+      )
+    `);
+  }
+}

package/src/utils/pgwire_utils.ts

@@ -0,0 +1,73 @@
+// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
+
+import * as pgwire from '@powersync/service-jpgwire';
+import { SqliteJsonValue, SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
+
+import { logger } from '@powersync/lib-services-framework';
+
+/**
+ * pgwire message -> SQLite row.
+ * @param message
+ */
+export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteRow {
+  const rawData = (message as any).afterRaw;
+
+  const record = pgwire.decodeTuple(message.relation, rawData);
+  return toSyncRulesRow(record);
+}
+
+/**
+ * pgwire message -> SQLite row.
+ * @param message
+ */
+export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteRow | undefined {
+  const rawData = (message as any).beforeRaw;
+  if (rawData == null) {
+    return undefined;
+  }
+  const record = pgwire.decodeTuple(message.relation, rawData);
+  return toSyncRulesRow(record);
+}
+
+export function escapeIdentifier(identifier: string) {
+  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
+
+export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam {
+  if (arg == null) {
+    return { type: 'varchar', value: null };
+  } else if (typeof arg == 'string') {
+    return { type: 'varchar', value: arg };
+  } else if (typeof arg == 'number') {
+    if (Number.isInteger(arg)) {
+      return { type: 'int8', value: arg };
+    } else {
+      return { type: 'float8', value: arg };
+    }
+  } else if (typeof arg == 'boolean') {
+    return { type: 'bool', value: arg };
+  } else if (typeof arg == 'bigint') {
+    return { type: 'int8', value: arg };
+  } else {
+    throw new Error(`Unsupported query parameter: ${typeof arg}`);
+  }
+}
+
+export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
+export async function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
+
+/**
+ * Retry a simple query - up to 2 attempts total.
+ */
+export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
+  for (let tries = 2; ; tries--) {
+    try {
+      return await db.query(...args);
+    } catch (e) {
+      if (tries == 1) {
+        throw e;
+      }
+      logger.warn('Query error, retrying', e);
+    }
+  }
+}

package/src/utils/populate_test_data.ts

@@ -0,0 +1,77 @@
+import * as crypto from 'crypto';
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
+
+import * as pgwire from '@powersync/service-jpgwire';
+
+// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
+
+export interface PopulateDataOptions {
+  connection: pgwire.NormalizedConnectionConfig;
+  num_transactions: number;
+  per_transaction: number;
+  size: number;
+}
+
+if (isMainThread || parentPort == null) {
+  // Not a worker - ignore
+} else {
+  try {
+    const options = workerData as PopulateDataOptions;
+
+    const result = await populateDataInner(options);
+    parentPort.postMessage(result);
+    process.exit(0);
+  } catch (e) {
+    // This is a bug, not a connection issue
+    console.error(e);
+    // Only closes the Worker thread
+    process.exit(2);
+  }
+}
+
+async function populateDataInner(options: PopulateDataOptions) {
+  // Dedicated connection so we can release the memory easily
+  const initialDb = await pgwire.connectPgWire(options.connection, { type: 'standard' });
+  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
+  let operation_count = 0;
+  for (let i = 0; i < options.num_transactions; i++) {
+    const prefix = `test${i}K`;
+
+    await initialDb.query({
+      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
+      params: [
+        { type: 'varchar', value: prefix },
+        { type: 'varchar', value: largeDescription },
+        { type: 'int4', value: options.per_transaction }
+      ]
+    });
+    operation_count += options.per_transaction;
+  }
+  await initialDb.end();
+  return operation_count;
+}
+
+export async function populateData(options: PopulateDataOptions) {
+  const WORKER_TIMEOUT = 30_000;
+
+  const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
+    workerData: options
+  });
+  const timeout = setTimeout(() => {
+    // Exits with code 1 below
+    worker.terminate();
+  }, WORKER_TIMEOUT);
+  try {
+    return await new Promise<number>((resolve, reject) => {
+      worker.on('message', resolve);
+      worker.on('error', reject);
+      worker.on('exit', (code) => {
+        if (code !== 0) {
+          reject(new Error(`Populating data failed with exit code ${code}`));
+        }
+      });
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/test/src/__snapshots__/pg_test.test.ts.snap

@@ -0,0 +1,256 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`pg data types > schema 1`] = `
+[
+  {
+    "name": "public",
+    "tables": [
+      {
+        "columns": [
+          {
+            "name": "id",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "text",
+            "pg_type": "text",
+            "type": "text",
+          },
+          {
+            "name": "uuid",
+            "pg_type": "uuid",
+            "type": "uuid",
+          },
+          {
+            "name": "varchar",
+            "pg_type": "varchar",
+            "type": "character varying(255)",
+          },
+          {
+            "name": "bool",
+            "pg_type": "bool",
+            "type": "boolean",
+          },
+          {
+            "name": "bytea",
+            "pg_type": "bytea",
+            "type": "bytea",
+          },
+          {
+            "name": "int2",
+            "pg_type": "int2",
+            "type": "smallint",
+          },
+          {
+            "name": "int4",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "int8",
+            "pg_type": "int8",
+            "type": "bigint",
+          },
+          {
+            "name": "float4",
+            "pg_type": "float4",
+            "type": "real",
+          },
+          {
+            "name": "float8",
+            "pg_type": "float8",
+            "type": "double precision",
+          },
+          {
+            "name": "numeric",
+            "pg_type": "numeric",
+            "type": "numeric",
+          },
+          {
+            "name": "json",
+            "pg_type": "json",
+            "type": "json",
+          },
+          {
+            "name": "jsonb",
+            "pg_type": "jsonb",
+            "type": "jsonb",
+          },
+          {
+            "name": "pg_lsn",
+            "pg_type": "pg_lsn",
+            "type": "pg_lsn",
+          },
+          {
+            "name": "date",
+            "pg_type": "date",
+            "type": "date",
+          },
+          {
+            "name": "time",
+            "pg_type": "time",
+            "type": "time without time zone",
+          },
+          {
+            "name": "timestamp",
+            "pg_type": "timestamp",
+            "type": "timestamp without time zone",
+          },
+          {
+            "name": "timestamptz",
+            "pg_type": "timestamptz",
+            "type": "timestamp with time zone",
+          },
+          {
+            "name": "interval",
+            "pg_type": "interval",
+            "type": "interval",
+          },
+          {
+            "name": "macaddr",
+            "pg_type": "macaddr",
+            "type": "macaddr",
+          },
+          {
+            "name": "inet",
+            "pg_type": "inet",
+            "type": "inet",
+          },
+          {
+            "name": "oid",
+            "pg_type": "oid",
+            "type": "oid",
+          },
+        ],
+        "name": "test_data",
+      },
+      {
+        "columns": [
+          {
+            "name": "id",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "text",
+            "pg_type": "text[]",
+            "type": "text[]",
+          },
+          {
+            "name": "uuid",
+            "pg_type": "uuid[]",
+            "type": "uuid[]",
+          },
+          {
+            "name": "varchar",
+            "pg_type": "varchar[]",
+            "type": "character varying(255)[]",
+          },
+          {
+            "name": "bool",
+            "pg_type": "bool[]",
+            "type": "boolean[]",
+          },
+          {
+            "name": "bytea",
+            "pg_type": "bytea[]",
+            "type": "bytea[]",
+          },
+          {
+            "name": "int2",
+            "pg_type": "int2[]",
+            "type": "smallint[]",
+          },
+          {
+            "name": "int4",
+            "pg_type": "int4[]",
+            "type": "integer[]",
+          },
+          {
+            "name": "int8",
+            "pg_type": "int8[]",
+            "type": "bigint[]",
+          },
+          {
+            "name": "float4",
+            "pg_type": "float4[]",
+            "type": "real[]",
+          },
+          {
+            "name": "float8",
+            "pg_type": "float8[]",
+            "type": "double precision[]",
+          },
+          {
+            "name": "numeric",
+            "pg_type": "numeric[]",
+            "type": "numeric[]",
+          },
+          {
+            "name": "json",
+            "pg_type": "json[]",
+            "type": "json[]",
+          },
+          {
+            "name": "jsonb",
+            "pg_type": "jsonb[]",
+            "type": "jsonb[]",
+          },
+          {
+            "name": "pg_lsn",
+            "pg_type": "pg_lsn[]",
+            "type": "pg_lsn[]",
+          },
+          {
+            "name": "date",
+            "pg_type": "date[]",
+            "type": "date[]",
+          },
+          {
+            "name": "time",
+            "pg_type": "time[]",
+            "type": "time without time zone[]",
+          },
+          {
+            "name": "timestamp",
+            "pg_type": "timestamp[]",
+            "type": "timestamp without time zone[]",
+          },
+          {
+            "name": "timestamptz",
+            "pg_type": "timestamptz[]",
+            "type": "timestamp with time zone[]",
+          },
+          {
+            "name": "interval",
+            "pg_type": "interval[]",
+            "type": "interval[]",
+          },
+          {
+            "name": "macaddr",
+            "pg_type": "macaddr[]",
+            "type": "macaddr[]",
+          },
+          {
+            "name": "inet",
+            "pg_type": "inet[]",
+            "type": "inet[]",
+          },
+          {
+            "name": "oid",
+            "pg_type": "oid[]",
+            "type": "oid[]",
+          },
+          {
+            "name": "multidimensional",
+            "pg_type": "text[]",
+            "type": "text[]",
+          },
+        ],
+        "name": "test_data_arrays",
+      },
+    ],
+  },
+]
+`;

package/test/src/env.ts

@@ -0,0 +1,7 @@
+import { utils } from '@powersync/lib-services-framework';
+
+export const env = utils.collectEnvironmentVariables({
+  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
+  CI: utils.type.boolean.default('false'),
+  SLOW_TESTS: utils.type.boolean.default('false')
+});

package/test/src/large_batch.test.ts

@@ -0,0 +1,195 @@
+import { MONGO_STORAGE_FACTORY, StorageFactory } from '@core-tests/util.js';
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { TEST_CONNECTION_OPTIONS } from './util.js';
+import { walStreamTest } from './wal_stream_utils.js';
+import { populateData } from '../../dist/utils/populate_test_data.js';
+
+describe('batch replication tests - mongodb', function () {
+  // These are slow but consistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineBatchTests(MONGO_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
+const BASIC_SYNC_RULES = `bucket_definitions:
+  global:
+    data:
+      - SELECT id, description, other FROM "test_data"`;
+
+function defineBatchTests(factory: StorageFactory) {
+  test(
+    'update large record',
+    walStreamTest(factory, async (context) => {
+      // This test generates a large transaction in MongoDB, despite the replicated data
+      // not being that large.
+      // If we don't limit transaction size, we could run into this error:
+      // > -31800: transaction is too large and will not fit in the storage engine cache
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      let operation_count = await populateData({
+        num_transactions: 1,
+        per_transaction: 80,
+        size: 4_000_000,
+        connection: TEST_CONNECTION_OPTIONS
+      });
+
+      const start = Date.now();
+
+      context.startStreaming();
+
+      const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+      const duration = Date.now() - start;
+      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+      expect(checksum.get('global[]')!.count).toEqual(operation_count);
+      const perSecond = Math.round((operation_count / duration) * 1000);
+      console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
+    }),
+    { timeout: 120_000 }
+  );
+
+  test(
+    'initial replication performance',
+    walStreamTest(factory, async (context) => {
+      // Manual test to check initial replication performance and memory usage
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      // Some stats (varies a lot):
+      // Old 'postgres' driver, using cursor(2)
+      // 15 ops in 19559ms 1 ops/s. 354MB RSS, 115MB heap, 137MB external
+      // 25 ops in 42984ms 1 ops/s. 377MB RSS, 129MB heap, 137MB external
+      // 35 ops in 41337ms 1 ops/s. 365MB RSS, 115MB heap, 137MB external
+
+      // streaming with pgwire
+      // 15 ops in 26423ms 1 ops/s. 379MB RSS, 128MB heap, 182MB external, 165MB ArrayBuffers
+      // 35 ops in 78897ms 0 ops/s. 539MB RSS, 52MB heap, 87MB external, 83MB ArrayBuffers
+
+      let operation_count = await populateData({
+        num_transactions: 1,
+        per_transaction: 35,
+        size: 14_000_000,
+        connection: TEST_CONNECTION_OPTIONS
+      });
+
+      global.gc?.();
+
+      // Note that we could already have high memory usage at this point
+      printMemoryUsage();
+
+      let interval = setInterval(() => {
+        printMemoryUsage();
+      }, 2000);
+      try {
+        const start = Date.now();
+
+        await context.replicateSnapshot();
+        await context.storage!.autoActivate();
+        context.startStreaming();
+
+        const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+        const duration = Date.now() - start;
+        const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+        expect(checksum.get('global[]')!.count).toEqual(operation_count);
+        const perSecond = Math.round((operation_count / duration) * 1000);
+        console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
+        printMemoryUsage();
+      } finally {
+        clearInterval(interval);
+      }
+    }),
+    { timeout: 120_000 }
+  );
+
+  test(
+    'large number of operations',
+    walStreamTest(factory, async (context) => {
+      // This just tests performance of a large number of operations inside a transaction.
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      const numTransactions = 20;
+      const perTransaction = 1500;
+      let operationCount = 0;
+
+      const description = 'description';
+
+      for (let i = 0; i < numTransactions; i++) {
+        const prefix = `test${i}K`;
+
+        await pool.query(
+          {
+            statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2 || i, 'foo' FROM generate_series(1, $3) i`,
+            params: [
+              { type: 'varchar', value: prefix },
+              { type: 'varchar', value: description },
+              { type: 'int4', value: perTransaction }
+            ]
+          },
+          {
+            statement: `UPDATE test_data SET other = other || '#' WHERE id LIKE $1 || '%'`,
+            params: [{ type: 'varchar', value: prefix }]
+          }
+        );
+        operationCount += perTransaction * 2;
+      }
+
+      const start = Date.now();
+
+      context.startStreaming();
+
+      const checkpoint = await context.getCheckpoint({ timeout: 50_000 });
+      const duration = Date.now() - start;
+      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+      expect(checksum.get('global[]')!.count).toEqual(operationCount);
+      const perSecond = Math.round((operationCount / duration) * 1000);
+      // This number depends on the test machine, so we keep the test significantly
+      // lower than expected numbers.
+      expect(perSecond).toBeGreaterThan(1000);
+      console.log(`${operationCount} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
+
+      // Truncating is fast (~10k ops/second).
+      // We'd need a really large set of data to actually run into limits when truncating,
+      // but we just test with the data we have here.
+      const truncateStart = Date.now();
+      await pool.query(`TRUNCATE test_data`);
+
+      const checkpoint2 = await context.getCheckpoint({ timeout: 20_000 });
+      const truncateDuration = Date.now() - truncateStart;
+
+      const checksum2 = await context.storage!.getChecksums(checkpoint2, ['global[]']);
+      const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
+      expect(truncateCount).toEqual(numTransactions * perTransaction);
+      const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
+      console.log(`Truncated ${truncateCount} ops in ${truncateDuration}ms ${truncatePerSecond} ops/s. ${used}MB heap`);
+    }),
+    { timeout: 90_000 }
+  );
+
+  function printMemoryUsage() {
+    const memoryUsage = process.memoryUsage();
+
+    const rss = Math.round(memoryUsage.rss / 1024 / 1024);
+    const heap = Math.round(memoryUsage.heapUsed / 1024 / 1024);
+    const external = Math.round(memoryUsage.external / 1024 / 1024);
+    const arrayBuffers = Math.round(memoryUsage.arrayBuffers / 1024 / 1024);
+    console.log(`${rss}MB RSS, ${heap}MB heap, ${external}MB external, ${arrayBuffers}MB ArrayBuffers`);
+  }
+}