@powersync/service-core 0.8.8 → 0.9.0

package/test/src/wal_stream.test.ts

@@ -1,319 +0,0 @@
-import * as crypto from 'crypto';
-import { describe, expect, test } from 'vitest';
-import { BucketStorageFactory } from '@/storage/BucketStorage.js';
-import { MONGO_STORAGE_FACTORY } from './util.js';
-import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { Metrics } from '@/metrics/Metrics.js';
-
-type StorageFactory = () => Promise<BucketStorageFactory>;
-
-const BASIC_SYNC_RULES = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-`;
-
-describe(
-  'wal stream - mongodb',
-  function () {
-    defineWalStreamTests(MONGO_STORAGE_FACTORY);
-  },
-  { timeout: 20_000 }
-);
-
-function defineWalStreamTests(factory: StorageFactory) {
-  test(
-    'replicating basic values',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description, num FROM "test_data"`);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num int8)`
-      );
-
-      await context.replicateSnapshot();
-
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(
-          `INSERT INTO test_data(description, num) VALUES('test1', 1152921504606846976) returning id as test_id`
-        )
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([
-        putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
-      ]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-      expect(endRowCount - startRowCount).toEqual(1);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-
-  test(
-    'replicating case sensitive table',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-      bucket_definitions:
-        global:
-          data:
-            - SELECT id, description FROM "test_DATA"
-      `);
-
-      await pool.query(`DROP TABLE IF EXISTS "test_DATA"`);
-      await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-      expect(endRowCount - startRowCount).toEqual(1);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-
-  test(
-    'replicating TOAST values',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-      bucket_definitions:
-        global:
-          data:
-            - SELECT id, name, description FROM "test_data"
-      `);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)`
-      );
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      // Must be > 8kb after compression
-      const largeDescription = crypto.randomBytes(20_000).toString('hex');
-      const [{ test_id }] = pgwireRows(
-        await pool.query({
-          statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`,
-          params: [{ type: 'varchar', value: largeDescription }]
-        })
-      );
-
-      await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`);
-
-      const data = await context.getBucketData('global[]');
-      expect(data.slice(0, 1)).toMatchObject([
-        putOp('test_data', { id: test_id, name: 'test1', description: largeDescription })
-      ]);
-      expect(data.slice(1)).toMatchObject([
-        putOp('test_data', { id: test_id, name: 'test2', description: largeDescription })
-      ]);
-    })
-  );
-
-  test(
-    'replicating TRUNCATE',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-  by_test_data:
-    parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
-    data: []
-`;
-      await context.updateSyncRules(syncRuleContent);
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-      await pool.query(`TRUNCATE test_data`);
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([
-        putOp('test_data', { id: test_id, description: 'test1' }),
-        removeOp('test_data', test_id)
-      ]);
-    })
-  );
-
-  test(
-    'replicating changing primary key',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const [{ test_id: test_id2 }] = pgwireRows(
-        await pool.query(
-          `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id`
-        )
-      );
-
-      // This update may fail replicating with:
-      // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
-      await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`);
-
-      // Re-use old id again
-      await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`);
-      await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`);
-
-      const data = await context.getBucketData('global[]');
-      expect(data).toMatchObject([
-        // Initial insert
-        putOp('test_data', { id: test_id, description: 'test1' }),
-        // Update id, then description
-        removeOp('test_data', test_id),
-        putOp('test_data', { id: test_id2, description: 'test2a' }),
-        putOp('test_data', { id: test_id2, description: 'test2b' }),
-        // Re-use old id
-        putOp('test_data', { id: test_id, description: 'test1b' }),
-        putOp('test_data', { id: test_id, description: 'test1c' })
-      ]);
-    })
-  );
-
-  test(
-    'initial sync',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const data = await context.getBucketData('global[]');
-      expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
-    })
-  );
-
-  test(
-    'record too large',
-    walStreamTest(factory, async (context) => {
-      await context.updateSyncRules(`bucket_definitions:
-      global:
-        data:
-          - SELECT id, description, other FROM "test_data"`);
-      const { pool } = context;
-
-      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-      await context.replicateSnapshot();
-
-      // 4MB
-      const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
-      // 18MB
-      const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
-
-      await pool.query({
-        statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
-        params: [{ type: 'varchar', value: tooLargeDescription }]
-      });
-      await pool.query({
-        statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
-        params: [{ type: 'varchar', value: largeDescription }]
-      });
-
-      context.startStreaming();
-
-      const data = await context.getBucketData('global[]');
-      expect(data.length).toEqual(1);
-      const row = JSON.parse(data[0].data as string);
-      delete row.description;
-      expect(row).toEqual({ id: 't1', other: 'foo' });
-      delete data[0].data;
-      expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
-    })
-  );
-
-  test(
-    'table not in sync rules',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-
-      await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-
-      const startRowCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([]);
-      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount =
-        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      // There was a transaction, but we should not replicate any actual data
-      expect(endRowCount - startRowCount).toEqual(0);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-}

package/test/src/wal_stream_utils.ts

@@ -1,156 +0,0 @@
-import * as pgwire from '@powersync/service-jpgwire';
-import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
-import { BucketStorageFactory, SyncRulesBucketStorage } from '../../src/storage/BucketStorage.js';
-import { OplogEntry } from '../../src/util/protocol-types.js';
-import { getClientCheckpoint } from '../../src/util/utils.js';
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
-import { PgManager } from '../../src/util/PgManager.js';
-import { JSONBig } from '@powersync/service-jsonbig';
-
-/**
- * Tests operating on the wal stream need to configure the stream and manage asynchronous
- * replication, which gets a little tricky.
- *
- * This wraps a test in a function that configures all the context, and tears it down afterwards.
- */
-export function walStreamTest(
-  factory: () => Promise<BucketStorageFactory>,
-  test: (context: WalStreamTestContext) => Promise<void>
-): () => Promise<void> {
-  return async () => {
-    const f = await factory();
-    const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-
-    await clearTestDb(connections.pool);
-    const context = new WalStreamTestContext(f, connections);
-    try {
-      await test(context);
-    } finally {
-      await context.dispose();
-    }
-  };
-}
-
-export class WalStreamTestContext {
-  private _walStream?: WalStream;
-  private abortController = new AbortController();
-  private streamPromise?: Promise<void>;
-  public storage?: SyncRulesBucketStorage;
-  private replicationConnection?: pgwire.PgConnection;
-
-  constructor(public factory: BucketStorageFactory, public connections: PgManager) {}
-
-  async dispose() {
-    this.abortController.abort();
-    await this.streamPromise;
-    this.connections.destroy();
-  }
-
-  get pool() {
-    return this.connections.pool;
-  }
-
-  async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules({ content: content });
-    this.storage = this.factory.getInstance(syncRules.parsed());
-    return this.storage!;
-  }
-
-  get walStream() {
-    if (this.storage == null) {
-      throw new Error('updateSyncRules() first');
-    }
-    if (this._walStream) {
-      return this._walStream;
-    }
-    const options: WalStreamOptions = {
-      storage: this.storage,
-      factory: this.factory,
-      connections: this.connections,
-      abort_signal: this.abortController.signal
-    };
-    this._walStream = new WalStream(options);
-    return this._walStream!;
-  }
-
-  async replicateSnapshot() {
-    this.replicationConnection = await this.connections.replicationConnection();
-    await this.walStream.initReplication(this.replicationConnection);
-    await this.storage!.autoActivate();
-  }
-
-  startStreaming() {
-    if (this.replicationConnection == null) {
-      throw new Error('Call replicateSnapshot() before startStreaming()');
-    }
-    this.streamPromise = this.walStream.streamChanges(this.replicationConnection!);
-  }
-
-  async getCheckpoint(options?: { timeout?: number }) {
-    let checkpoint = await Promise.race([
-      getClientCheckpoint(this.connections.pool, this.factory, { timeout: options?.timeout ?? 15_000 }),
-      this.streamPromise
-    ]);
-    if (typeof checkpoint == undefined) {
-      // This indicates an issue with the test setup - streamingPromise completed instead
-      // of getClientCheckpoint()
-      throw new Error('Test failure - streamingPromise completed');
-    }
-    return checkpoint as string;
-  }
-
-  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
-    let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, string>(Object.entries(buckets));
-    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
-  }
-
-  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
-    start ??= '0';
-    let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, string>([[bucket, start]]);
-    const batch = await this.storage!.getBucketDataBatch(checkpoint, map);
-    const batches = await fromAsync(batch);
-    return batches[0]?.batch.data ?? [];
-  }
-}
-
-export function putOp(table: string, data: Record<string, any>): Partial<OplogEntry> {
-  return {
-    op: 'PUT',
-    object_type: table,
-    object_id: data.id,
-    data: JSONBig.stringify(data)
-  };
-}
-
-export function removeOp(table: string, id: string): Partial<OplogEntry> {
-  return {
-    op: 'REMOVE',
-    object_type: table,
-    object_id: id
-  };
-}
-
-export function compareIds(a: OplogEntry, b: OplogEntry) {
-  return a.object_id!.localeCompare(b.object_id!);
-}
-
-export async function fromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> {
-  const items: T[] = [];
-  for await (const item of source) {
-    items.push(item);
-  }
-  return items;
-}
-
-export async function oneFromAsync<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T> {
-  const items: T[] = [];
-  for await (const item of source) {
-    items.push(item);
-  }
-  if (items.length != 1) {
-    throw new Error(`One item expected, got: ${items.length}`);
-  }
-  return items[0];
-}