@powersync/service-module-postgres 0.0.0-dev-20240918092408

package/test/src/wal_stream.test.ts

@@ -0,0 +1,319 @@
+import { putOp, removeOp } from '@core-tests/stream_utils.js';
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { BucketStorageFactory, Metrics } from '@powersync/service-core';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import * as crypto from 'crypto';
+import { describe, expect, test } from 'vitest';
+import { walStreamTest } from './wal_stream_utils.js';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+
+describe(
+  'wal stream - mongodb',
+  function () {
+    defineWalStreamTests(MONGO_STORAGE_FACTORY);
+  },
+  { timeout: 20_000 }
+);
+
+function defineWalStreamTests(factory: StorageFactory) {
+  test(
+    'replicating basic values',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description, num FROM "test_data"`);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num int8)`
+      );
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(
+          `INSERT INTO test_data(description, num) VALUES('test1', 1152921504606846976) returning id as test_id`
+        )
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
+      ]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'replicating case sensitive table',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, description FROM "test_DATA"
+      `);
+
+      await pool.query(`DROP TABLE IF EXISTS "test_DATA"`);
+      await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'replicating TOAST values',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, name, description FROM "test_data"
+      `);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)`
+      );
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      // Must be > 8kb after compression
+      const largeDescription = crypto.randomBytes(20_000).toString('hex');
+      const [{ test_id }] = pgwireRows(
+        await pool.query({
+          statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`,
+          params: [{ type: 'varchar', value: largeDescription }]
+        })
+      );
+
+      await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`);
+
+      const data = await context.getBucketData('global[]');
+      expect(data.slice(0, 1)).toMatchObject([
+        putOp('test_data', { id: test_id, name: 'test1', description: largeDescription })
+      ]);
+      expect(data.slice(1)).toMatchObject([
+        putOp('test_data', { id: test_id, name: 'test2', description: largeDescription })
+      ]);
+    })
+  );
+
+  test(
+    'replicating TRUNCATE',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+  by_test_data:
+    parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
+    data: []
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+      await pool.query(`TRUNCATE test_data`);
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id, description: 'test1' }),
+        removeOp('test_data', test_id)
+      ]);
+    })
+  );
+
+  test(
+    'replicating changing primary key',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const [{ test_id: test_id2 }] = pgwireRows(
+        await pool.query(
+          `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id`
+        )
+      );
+
+      // This update may fail replicating with:
+      // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
+      await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`);
+
+      // Re-use old id again
+      await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`);
+      await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`);
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([
+        // Initial insert
+        putOp('test_data', { id: test_id, description: 'test1' }),
+        // Update id, then description
+        removeOp('test_data', test_id),
+        putOp('test_data', { id: test_id2, description: 'test2a' }),
+        putOp('test_data', { id: test_id2, description: 'test2b' }),
+        // Re-use old id
+        putOp('test_data', { id: test_id, description: 'test1b' }),
+        putOp('test_data', { id: test_id, description: 'test1c' })
+      ]);
+    })
+  );
+
+  test(
+    'initial sync',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    })
+  );
+
+  test(
+    'record too large',
+    walStreamTest(factory, async (context) => {
+      await context.updateSyncRules(`bucket_definitions:
+      global:
+        data:
+          - SELECT id, description, other FROM "test_data"`);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      // 4MB
+      const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
+      // 18MB
+      const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
+
+      await pool.query({
+        statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
+        params: [{ type: 'varchar', value: tooLargeDescription }]
+      });
+      await pool.query({
+        statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
+        params: [{ type: 'varchar', value: largeDescription }]
+      });
+
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data.length).toEqual(1);
+      const row = JSON.parse(data[0].data as string);
+      delete row.description;
+      expect(row).toEqual({ id: 't1', other: 'foo' });
+      delete data[0].data;
+      expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
+    })
+  );
+
+  test(
+    'table not in sync rules',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      // There was a transaction, but we should not replicate any actual data
+      expect(endRowCount - startRowCount).toEqual(0);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+}

package/test/src/wal_stream_utils.ts

@@ -0,0 +1,121 @@
+import { BucketStorageFactory, SyncRulesBucketStorage } from '@powersync/service-core';
+import * as pgwire from '@powersync/service-jpgwire';
+import { TEST_CONNECTION_OPTIONS, clearTestDb, getClientCheckpoint } from './util.js';
+import { WalStream, WalStreamOptions, PUBLICATION_NAME } from '@module/replication/WalStream.js';
+import { fromAsync } from '@core-tests/stream_utils.js';
+import { PgManager } from '@module/replication/PgManager.js';
+
+/**
+ * Tests operating on the wal stream need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps a test in a function that configures all the context, and tears it down afterwards.
+ */
+export function walStreamTest(
+  factory: () => Promise<BucketStorageFactory>,
+  test: (context: WalStreamTestContext) => Promise<void>
+): () => Promise<void> {
+  return async () => {
+    const f = await factory();
+    const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
+
+    await clearTestDb(connectionManager.pool);
+    const context = new WalStreamTestContext(f, connectionManager);
+    try {
+      await test(context);
+    } finally {
+      await context.dispose();
+    }
+  };
+}
+
+export class WalStreamTestContext {
+  private _walStream?: WalStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+  private replicationConnection?: pgwire.PgConnection;
+
+  constructor(public factory: BucketStorageFactory, public connectionManager: PgManager) {}
+
+  async dispose() {
+    this.abortController.abort();
+    await this.streamPromise;
+    await this.connectionManager.destroy();
+  }
+
+  get pool() {
+    return this.connectionManager.pool;
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  get publicationName() {
+    return PUBLICATION_NAME;
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules({ content: content });
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  get walStream() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._walStream) {
+      return this._walStream;
+    }
+    const options: WalStreamOptions = {
+      storage: this.storage,
+      connections: this.connectionManager,
+      abort_signal: this.abortController.signal
+    };
+    this._walStream = new WalStream(options);
+    return this._walStream!;
+  }
+
+  async replicateSnapshot() {
+    this.replicationConnection = await this.connectionManager.replicationConnection();
+    await this.walStream.initReplication(this.replicationConnection);
+    await this.storage!.autoActivate();
+  }
+
+  startStreaming() {
+    if (this.replicationConnection == null) {
+      throw new Error('Call replicateSnapshot() before startStreaming()');
+    }
+    this.streamPromise = this.walStream.streamChanges(this.replicationConnection!);
+  }
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(this.pool, this.factory, { timeout: options?.timeout ?? 15_000 }),
+      this.streamPromise
+    ]);
+    if (typeof checkpoint == undefined) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint as string;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>(Object.entries(buckets));
+    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
+    start ??= '0';
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>([[bucket, start]]);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.batch.data ?? [];
+  }
+}

package/test/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "baseUrl": "./",
+    "noEmit": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["../../../packages/service-core/src/*"],
+      "@module/*": ["../src/*"],
+      "@core-tests/*": ["../../../packages/service-core/test/src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    },
+    {
+      "path": "../../../packages/service-core/test"
+    },
+    {
+      "path": "../../../packages/service-core/"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/types"
+    },
+    {
+      "path": "../../packages/jsonbig"
+    },
+    {
+      "path": "../../packages/jpgwire"
+    },
+    {
+      "path": "../../packages/sync-rules"
+    },
+    {
+      "path": "../../packages/service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
+    }
+  ]
+}