@powersync/service-core 0.0.2

package/test/src/util.ts

@@ -0,0 +1,67 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { normalizeConnection } from '@powersync/service-types';
+import * as mongo from 'mongodb';
+import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
+import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
+import { PowerSyncMongo } from '../../src/storage/mongo/db.js';
+import { escapeIdentifier } from '../../src/util/pgwire_utils.js';
+import { env } from './env.js';
+
+export const TEST_URI = env.PG_TEST_URL;
+
+export type StorageFactory = () => Promise<BucketStorageFactory>;
+
+export const MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+  const db = await connectMongo();
+  await db.clear();
+  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
+};
+
+export async function clearTestDb(db: pgwire.PgClient) {
+  await db.query(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
+  try {
+    await db.query(`DROP PUBLICATION powersync`);
+  } catch (e) {
+    // Ignore
+  }
+
+  await db.query(`CREATE PUBLICATION powersync FOR ALL TABLES`);
+
+  const tableRows = pgwire.pgwireRows(
+    await db.query(`SELECT table_name FROM information_schema.tables where table_schema = 'public'`)
+  );
+  for (let row of tableRows) {
+    const name = row.table_name;
+    if (name.startsWith('test_')) {
+      await db.query(`DROP TABLE public.${escapeIdentifier(name)}`);
+    }
+  }
+}
+
+export const TEST_CONNECTION_OPTIONS = normalizeConnection({
+  type: 'postgresql',
+  uri: TEST_URI,
+  sslmode: 'disable'
+});
+
+export async function connectPgWire(type?: 'replication' | 'standard') {
+  const db = await pgwire.connectPgWire(TEST_CONNECTION_OPTIONS, { type });
+  return db;
+}
+
+export function connectPgPool() {
+  const db = pgwire.connectPgWirePool(TEST_CONNECTION_OPTIONS);
+  return db;
+}
+
+export async function connectMongo() {
+  // Short timeout for tests, to fail fast when the server is not available.
+  // Slightly longer timeouts for CI, to avoid arbitrary test failures
+  const client = new mongo.MongoClient(env.MONGO_TEST_URL, {
+    connectTimeoutMS: env.CI ? 15_000 : 5_000,
+    socketTimeoutMS: env.CI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500
+  });
+  const db = new PowerSyncMongo(client);
+  return db;
+}

package/test/src/validation.test.ts

@@ -0,0 +1,63 @@
+import { expect, test } from 'vitest';
+import { MONGO_STORAGE_FACTORY } from './util.js';
+import { walStreamTest } from './wal_stream_utils.js';
+import { WalConnection } from '../../src/replication/WalConnection.js';
+
+// Not quite a walStreamTest, but it helps to manage the connection
+test(
+  'validate tables',
+  walStreamTest(MONGO_STORAGE_FACTORY, async (context) => {
+    const { pool } = context;
+
+    await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+    const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+      - SELECT * FROM "other"
+      - SELECT * FROM "other%"
+`;
+
+    const syncRules = await context.factory.updateSyncRules({ content: syncRuleContent });
+
+    const walConnection = new WalConnection({
+      db: pool,
+      sync_rules: syncRules.parsed().sync_rules
+    });
+
+    const tablePatterns = syncRules.parsed().sync_rules.getSourceTables();
+    const tableInfo = await walConnection.getDebugTablesInfo(tablePatterns);
+    expect(tableInfo).toEqual([
+      {
+        schema: 'public',
+        pattern: 'test_data',
+        wildcard: false,
+        table: {
+          schema: 'public',
+          name: 'test_data',
+          replication_id: ['id'],
+          pattern: undefined,
+          data_queries: true,
+          parameter_queries: false,
+          errors: []
+        }
+      },
+      {
+        schema: 'public',
+        pattern: 'other',
+        wildcard: false,
+        table: {
+          schema: 'public',
+          name: 'other',
+          replication_id: [],
+          data_queries: true,
+          parameter_queries: false,
+          errors: [{ level: 'warning', message: 'Table "public"."other" not found.' }]
+        }
+      },
+      { schema: 'public', pattern: 'other%', wildcard: true, tables: [] }
+    ]);
+  })
+);

package/test/src/wal_stream.test.ts

@@ -0,0 +1,310 @@
+import * as crypto from 'crypto';
+import { describe, expect, test } from 'vitest';
+import { BucketStorageFactory } from '../../src/storage/BucketStorage.js';
+import { MONGO_STORAGE_FACTORY } from './util.js';
+import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
+import { pgwireRows } from '@powersync/service-jpgwire';
+import { getMetricValueForTests } from '../../src/metrics/metrics.js';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+
+describe(
+  'wal stream - mongodb',
+  function () {
+    defineWalStreamTests(MONGO_STORAGE_FACTORY);
+  },
+  { timeout: 20_000 }
+);
+
+function defineWalStreamTests(factory: StorageFactory) {
+  test(
+    'replicating basic values',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description, num FROM "test_data"`);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num int8)`
+      );
+
+      await context.replicateSnapshot();
+
+      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(
+          `INSERT INTO test_data(description, num) VALUES('test1', 1152921504606846976) returning id as test_id`
+        )
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
+      ]);
+      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'replicating case sensitive table',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, description FROM "test_DATA"
+      `);
+
+      await pool.query(`DROP TABLE IF EXISTS "test_DATA"`);
+      await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
+      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'replicating TOAST values',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, name, description FROM "test_data"
+      `);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)`
+      );
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      // Must be > 8kb after compression
+      const largeDescription = crypto.randomBytes(20_000).toString('hex');
+      const [{ test_id }] = pgwireRows(
+        await pool.query({
+          statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`,
+          params: [{ type: 'varchar', value: largeDescription }]
+        })
+      );
+
+      await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`);
+
+      const data = await context.getBucketData('global[]');
+      expect(data.slice(0, 1)).toMatchObject([
+        putOp('test_data', { id: test_id, name: 'test1', description: largeDescription })
+      ]);
+      expect(data.slice(1)).toMatchObject([
+        putOp('test_data', { id: test_id, name: 'test2', description: largeDescription })
+      ]);
+    })
+  );
+
+  test(
+    'replicating TRUNCATE',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      const syncRuleContent = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+  by_test_data:
+    parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
+    data: []
+`;
+      await context.updateSyncRules(syncRuleContent);
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+      await pool.query(`TRUNCATE test_data`);
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([
+        putOp('test_data', { id: test_id, description: 'test1' }),
+        removeOp('test_data', test_id)
+      ]);
+    })
+  );
+
+  test(
+    'replicating changing primary key',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const [{ test_id: test_id2 }] = pgwireRows(
+        await pool.query(
+          `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id`
+        )
+      );
+
+      // This update may fail replicating with:
+      // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
+      await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`);
+
+      // Re-use old id again
+      await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`);
+      await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`);
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([
+        // Initial insert
+        putOp('test_data', { id: test_id, description: 'test1' }),
+        // Update id, then description
+        removeOp('test_data', test_id),
+        putOp('test_data', { id: test_id2, description: 'test2a' }),
+        putOp('test_data', { id: test_id2, description: 'test2b' }),
+        // Re-use old id
+        putOp('test_data', { id: test_id, description: 'test1b' }),
+        putOp('test_data', { id: test_id, description: 'test1c' })
+      ]);
+    })
+  );
+
+  test(
+    'initial sync',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await pool.query(`DROP TABLE IF EXISTS test_data`);
+      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+      );
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
+    })
+  );
+
+  test(
+    'record too large',
+    walStreamTest(factory, async (context) => {
+      await context.updateSyncRules(`bucket_definitions:
+      global:
+        data:
+          - SELECT id, description, other FROM "test_data"`);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      // 4MB
+      const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
+      // 18MB
+      const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
+
+      await pool.query({
+        statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
+        params: [{ type: 'varchar', value: tooLargeDescription }]
+      });
+      await pool.query({
+        statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
+        params: [{ type: 'varchar', value: largeDescription }]
+      });
+
+      context.startStreaming();
+
+      const data = await context.getBucketData('global[]');
+      expect(data.length).toEqual(1);
+      const row = JSON.parse(data[0].data as string);
+      delete row.description;
+      expect(row).toEqual({ id: 't1', other: 'foo' });
+      delete data[0].data;
+      expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
+    })
+  );
+
+  test(
+    'table not in sync rules',
+    walStreamTest(factory, async (context) => {
+      const { pool } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const [{ test_id }] = pgwireRows(
+        await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`)
+      );
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([]);
+      const endRowCount = (await getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount = (await getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      // There was a transaction, but we should not replicate any actual data
+      expect(endRowCount - startRowCount).toEqual(0);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+}

package/test/src/wal_stream_utils.ts

@@ -0,0 +1,147 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
+import { BucketStorageFactory, SyncRulesBucketStorage } from '../../src/storage/BucketStorage.js';
+import { OplogEntry } from '../../src/util/protocol-types.js';
+import { getClientCheckpoint } from '../../src/util/utils.js';
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+import { PgManager } from '../../src/util/PgManager.js';
+import { JSONBig } from '@powersync/service-jsonbig';
+
+/**
+ * Tests operating on the wal stream need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps a test in a function that configures all the context, and tears it down afterwards.
+ */
+export function walStreamTest(
+  factory: () => Promise<BucketStorageFactory>,
+  test: (context: WalStreamTestContext) => Promise<void>
+): () => Promise<void> {
+  return async () => {
+    const f = await factory();
+    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
+    await connections.pool.query(
+      'select pg_drop_replication_slot(slot_name) from pg_replication_slots where active = false'
+    );
+    await clearTestDb(connections.pool);
+    const context = new WalStreamTestContext(f, connections);
+    try {
+      await test(context);
+    } finally {
+      await context.dispose();
+    }
+  };
+}
+
+export class WalStreamTestContext {
+  private _walStream?: WalStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+  private replicationConnection?: pgwire.PgConnection;
+
+  constructor(public factory: BucketStorageFactory, public connections: PgManager) {}
+
+  async dispose() {
+    this.abortController.abort();
+    await this.streamPromise;
+    this.connections.destroy();
+  }
+
+  get pool() {
+    return this.connections.pool;
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules({ content: content });
+    this.storage = this.factory.getInstance(syncRules.parsed());
+    return this.storage!;
+  }
+
+  get walStream() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._walStream) {
+      return this._walStream;
+    }
+    const options: WalStreamOptions = {
+      storage: this.storage,
+      factory: this.factory,
+      connections: this.connections,
+      abort_signal: this.abortController.signal
+    };
+    this._walStream = new WalStream(options);
+    return this._walStream!;
+  }
+
+  async replicateSnapshot() {
+    this.replicationConnection = await this.connections.replicationConnection();
+    await this.walStream.initReplication(this.replicationConnection);
+    await this.storage!.autoActivate();
+  }
+
+  startStreaming() {
+    if (this.replicationConnection == null) {
+      throw new Error('Call replicateSnapshot() before startStreaming()');
+    }
+    this.streamPromise = this.walStream.streamChanges(this.replicationConnection!);
+  }
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(this.connections.pool, this.factory, { timeout: options?.timeout ?? 15_000 }),
+      this.streamPromise
+    ]);
+    if (typeof checkpoint == undefined) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint as string;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>(Object.entries(buckets));
+    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
+    start ??= '0';
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>([[bucket, start]]);
+    const batch = await this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.data ?? [];
+  }
+}
+
+export function putOp(table: string, data: Record<string, any>): Partial<OplogEntry> {
+  return {
+    op: 'PUT',
+    object_type: table,
+    object_id: data.id,
+    data: JSONBig.stringify(data)
+  };
+}
+
+export function removeOp(table: string, id: string): Partial<OplogEntry> {
+  return {
+    op: 'REMOVE',
+    object_type: table,
+    object_id: id
+  };
+}
+
+export function compareIds(a: OplogEntry, b: OplogEntry) {
+  return a.object_id!.localeCompare(b.object_id!);
+}
+
+export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> {
+  const items: T[] = [];
+  for await (const item of source) {
+    items.push(item);
+  }
+  return items;
+}

package/test/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "@journeyapps-platform/micro-dev/tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "noEmit": true,
+    "baseUrl": "./",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["../src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "@journeyapps-platform/micro-dev/tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "baseUrl": ".",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../types"
+    }
+  ]
+}