@powersync/service-core 0.0.0-dev-20240620165206

package/test/src/env.ts

@@ -0,0 +1,8 @@
+import { utils } from '@powersync/lib-services-framework';
+
+export const env = utils.collectEnvironmentVariables({
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
+  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
+  CI: utils.type.boolean.default('false'),
+  SLOW_TESTS: utils.type.boolean.default('false')
+});

package/test/src/large_batch.test.ts

@@ -0,0 +1,194 @@
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { MONGO_STORAGE_FACTORY, StorageFactory, TEST_CONNECTION_OPTIONS } from './util.js';
+import { walStreamTest } from './wal_stream_utils.js';
+import { populateData } from '../../dist/util/populate_test_data.js';
+
+describe('batch replication tests - mongodb', function () {
+  // These are slow but consistent tests.
+  // Not run on every test run, but we do run on CI, or when manually debugging issues.
+  if (env.CI || env.SLOW_TESTS) {
+    defineBatchTests(MONGO_STORAGE_FACTORY);
+  } else {
+    // Need something in this file.
+    test('no-op', () => {});
+  }
+});
+
+const BASIC_SYNC_RULES = `bucket_definitions:
+  global:
+    data:
+      - SELECT id, description, other FROM "test_data"`;
+
+function defineBatchTests(factory: StorageFactory) {
+  test(
+    'update large record',
+    walStreamTest(factory, async (context) => {
+      // This test generates a large transaction in MongoDB, despite the replicated data
+      // not being that large.
+      // If we don't limit transaction size, we could run into this error:
+      // > -31800: transaction is too large and will not fit in the storage engine cache
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      let operation_count = await populateData({
+        num_transactions: 1,
+        per_transaction: 80,
+        size: 4_000_000,
+        connection: TEST_CONNECTION_OPTIONS
+      });
+
+      const start = Date.now();
+
+      context.startStreaming();
+
+      const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+      const duration = Date.now() - start;
+      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+      expect(checksum.get('global[]')!.count).toEqual(operation_count);
+      const perSecond = Math.round((operation_count / duration) * 1000);
+      console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
+    }),
+    { timeout: 120_000 }
+  );
+
+  test(
+    'initial replication performance',
+    walStreamTest(factory, async (context) => {
+      // Manual test to check initial replication performance and memory usage
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      // Some stats (varies a lot):
+      // Old 'postgres' driver, using cursor(2)
+      // 15 ops in 19559ms 1 ops/s. 354MB RSS, 115MB heap, 137MB external
+      // 25 ops in 42984ms 1 ops/s. 377MB RSS, 129MB heap, 137MB external
+      // 35 ops in 41337ms 1 ops/s. 365MB RSS, 115MB heap, 137MB external
+
+      // streaming with pgwire
+      // 15 ops in 26423ms 1 ops/s. 379MB RSS, 128MB heap, 182MB external, 165MB ArrayBuffers
+      // 35 ops in 78897ms 0 ops/s. 539MB RSS, 52MB heap, 87MB external, 83MB ArrayBuffers
+
+      let operation_count = await populateData({
+        num_transactions: 1,
+        per_transaction: 35,
+        size: 14_000_000,
+        connection: TEST_CONNECTION_OPTIONS
+      });
+
+      global.gc?.();
+
+      // Note that we could already have high memory usage at this point
+      printMemoryUsage();
+
+      let interval = setInterval(() => {
+        printMemoryUsage();
+      }, 2000);
+      try {
+        const start = Date.now();
+
+        await context.replicateSnapshot();
+        await context.storage!.autoActivate();
+        context.startStreaming();
+
+        const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
+        const duration = Date.now() - start;
+        const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+        expect(checksum.get('global[]')!.count).toEqual(operation_count);
+        const perSecond = Math.round((operation_count / duration) * 1000);
+        console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
+        printMemoryUsage();
+      } finally {
+        clearInterval(interval);
+      }
+    }),
+    { timeout: 120_000 }
+  );
+
+  test(
+    'large number of operations',
+    walStreamTest(factory, async (context) => {
+      // This just tests performance of a large number of operations inside a transaction.
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+      const { pool } = context;
+
+      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+
+      await context.replicateSnapshot();
+
+      const numTransactions = 20;
+      const perTransaction = 1500;
+      let operationCount = 0;
+
+      const description = 'description';
+
+      for (let i = 0; i < numTransactions; i++) {
+        const prefix = `test${i}K`;
+
+        await pool.query(
+          {
+            statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2 || i, 'foo' FROM generate_series(1, $3) i`,
+            params: [
+              { type: 'varchar', value: prefix },
+              { type: 'varchar', value: description },
+              { type: 'int4', value: perTransaction }
+            ]
+          },
+          {
+            statement: `UPDATE test_data SET other = other || '#' WHERE id LIKE $1 || '%'`,
+            params: [{ type: 'varchar', value: prefix }]
+          }
+        );
+        operationCount += perTransaction * 2;
+      }
+
+      const start = Date.now();
+
+      context.startStreaming();
+
+      const checkpoint = await context.getCheckpoint({ timeout: 50_000 });
+      const duration = Date.now() - start;
+      const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+      const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
+      expect(checksum.get('global[]')!.count).toEqual(operationCount);
+      const perSecond = Math.round((operationCount / duration) * 1000);
+      // This number depends on the test machine, so we keep the test significantly
+      // lower than expected numbers.
+      expect(perSecond).toBeGreaterThan(1000);
+      console.log(`${operationCount} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
+
+      // Truncating is fast (~10k ops/second).
+      // We'd need a really large set of data to actually run into limits when truncating,
+      // but we just test with the data we have here.
+      const truncateStart = Date.now();
+      await pool.query(`TRUNCATE test_data`);
+
+      const checkpoint2 = await context.getCheckpoint({ timeout: 20_000 });
+      const truncateDuration = Date.now() - truncateStart;
+
+      const checksum2 = await context.storage!.getChecksums(checkpoint2, ['global[]']);
+      const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
+      expect(truncateCount).toEqual(numTransactions * perTransaction);
+      const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
+      console.log(`Truncated ${truncateCount} ops in ${truncateDuration}ms ${truncatePerSecond} ops/s. ${used}MB heap`);
+    }),
+    { timeout: 90_000 }
+  );
+
+  function printMemoryUsage() {
+    const memoryUsage = process.memoryUsage();
+
+    const rss = Math.round(memoryUsage.rss / 1024 / 1024);
+    const heap = Math.round(memoryUsage.heapUsed / 1024 / 1024);
+    const external = Math.round(memoryUsage.external / 1024 / 1024);
+    const arrayBuffers = Math.round(memoryUsage.arrayBuffers / 1024 / 1024);
+    console.log(`${rss}MB RSS, ${heap}MB heap, ${external}MB external, ${arrayBuffers}MB ArrayBuffers`);
+  }
+}

package/test/src/merge_iterable.test.ts

@@ -0,0 +1,355 @@
+import * as timers from 'timers/promises';
+import { describe, expect, test } from 'vitest';
+import {
+  FixedMergeAsyncIterable,
+  mergeAsyncIterables,
+  mergeAsyncIterablesNew,
+  mergeAsyncIterablesOld
+} from '../../src/sync/merge.js';
+
+type MergeIteratorFunction = <T>(source: AsyncIterable<T>[]) => AsyncIterable<T>;
+
+describe('merge iterables', () => {
+  makeTests((source) => mergeAsyncIterablesNew(source));
+});
+
+describe('merge iterables - legacy implementation', () => {
+  makeTests((source) => mergeAsyncIterablesOld(source));
+});
+
+function makeTests(mergeIterables: MergeIteratorFunction) {
+  test('iterable should not pre-emptively continue (1)', async () => {
+    // Test that the generator stops after the first yield
+    let afterYield = false;
+    let inFinally = false;
+    const a = async function* (): AsyncGenerator<number, void> {
+      try {
+        yield 1;
+        afterYield = true;
+        throw new Error('foobar');
+      } finally {
+        inFinally = true;
+      }
+    };
+
+    const merged = mergeIterables([a()]);
+    const iter = merged[Symbol.asyncIterator]();
+    const r = await iter.next();
+    expect(r).toEqual({ value: 1, done: false });
+    await iter.return!();
+    expect(afterYield).toEqual(false);
+    expect(inFinally).toEqual(true);
+  });
+
+  test('iterable should not pre-emptively continue (2)', async () => {
+    // Test that the generator stops after the first yield
+    // Equivalent to the above test, but using `for await` instead of manually iterating.
+    let afterYield = false;
+    let inFinally = false;
+    const a = async function* (): AsyncGenerator<number, void> {
+      try {
+        yield 1;
+        afterYield = true;
+        throw new Error('foobar');
+      } finally {
+        inFinally = true;
+      }
+    };
+
+    const merged = mergeIterables([a()]);
+
+    for await (let r of merged) {
+      expect(r).toEqual(1);
+      break;
+    }
+
+    expect(afterYield).toEqual(false);
+    expect(inFinally).toEqual(true);
+  });
+
+  test('iterable should cleanup', async () => {
+    // Test that the generator is exhauseted
+    let afterYield = false;
+    let inFinally = false;
+    const a = async function* (): AsyncGenerator<number, void> {
+      try {
+        yield 1;
+        afterYield = true;
+      } finally {
+        inFinally = true;
+      }
+    };
+
+    const merged = mergeIterables([a()]);
+
+    for await (let r of merged) {
+      expect(r).toEqual(1);
+    }
+    expect(afterYield).toEqual(true);
+    expect(inFinally).toEqual(true);
+  });
+
+  test('multiple iterables should cleanup', async () => {
+    // Test that the generator is exhauseted
+    let afterYieldA = false;
+    let inFinallyA = false;
+    let afterYieldB = false;
+    let inFinallyB = false;
+    const a = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'a';
+        afterYieldA = true;
+      } finally {
+        inFinallyA = true;
+      }
+    };
+    const b = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'b';
+        afterYieldB = true;
+      } finally {
+        inFinallyB = true;
+      }
+    };
+
+    const merged = mergeIterables([a(), b()]);
+
+    const iter = merged[Symbol.asyncIterator]();
+    expect(await iter.next()).toEqual({ value: 'a', done: false });
+    expect(await iter.next()).toEqual({ value: 'b', done: false });
+    expect(await iter.next()).toEqual({ value: undefined, done: true });
+
+    expect(afterYieldA).toEqual(true);
+    expect(inFinallyA).toEqual(true);
+    expect(afterYieldB).toEqual(true);
+    expect(inFinallyB).toEqual(true);
+  });
+
+  test('merging two (1)', async () => {
+    let aDone = false;
+    let bDone = false;
+    let aReached = false;
+    let bReached = false;
+
+    const a = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'a';
+        await timers.setTimeout(0);
+        // Should reach this
+        aReached = true;
+        throw new Error('a');
+      } finally {
+        // Should reach this
+        aDone = true;
+      }
+    };
+    const b = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'b';
+        // Should not reach this
+        bReached = true;
+        await timers.setTimeout(1);
+        throw new Error('b');
+      } finally {
+        // Should reach this
+        bDone = true;
+      }
+    };
+
+    const merged = mergeIterables([a(), b()]);
+
+    const iter = merged[Symbol.asyncIterator]();
+    expect(await iter.next()).toEqual({ value: 'a', done: false });
+    expect(await iter.next()).toEqual({ value: 'b', done: false });
+    await iter.return!();
+
+    await timers.setTimeout(2);
+
+    expect(aDone).toEqual(true);
+    expect(bDone).toEqual(true);
+    expect(aReached).toEqual(true);
+    expect(bReached).toEqual(false);
+  });
+
+  test('ending one iterator early', async () => {
+    let aDone = false;
+    let bDone = false;
+    let b2Reached = false;
+    let b3Reached = false;
+
+    const a = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'a';
+      } finally {
+        aDone = true;
+      }
+    };
+    const b = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'b1';
+        // This point should be reached, even if the value is not used
+        b2Reached = true;
+        yield 'b2';
+        // This point should not be reached
+        b3Reached = true;
+        yield 'b3';
+      } finally {
+        bDone = true;
+      }
+    };
+
+    const merged = mergeIterables([a(), b()]);
+
+    const iter = merged[Symbol.asyncIterator]();
+    expect(await iter.next()).toEqual({ value: 'a', done: false });
+    expect(await iter.next()).toEqual({ value: 'b1', done: false });
+    expect(await iter.next()).toEqual({ value: undefined, done: true });
+
+    await timers.setTimeout(2);
+
+    expect(aDone).toEqual(true);
+    expect(bDone).toEqual(true);
+    expect(b2Reached).toEqual(true);
+    expect(b3Reached).toEqual(false);
+  });
+
+  test('propagating errors from source', async () => {
+    // An error in one source iterator should:
+    // 1. Propagate to the merged iterator.
+    // 2. Return the other source iterator(s).
+    let bDone = false;
+    let bError = null;
+    let b2Reached = false;
+    let b3Reached = false;
+
+    const a = async function* (): AsyncGenerator<string, void> {
+      yield 'a';
+      await timers.setTimeout(0);
+      throw new Error('aError');
+    };
+
+    const b = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'b1';
+        await timers.setTimeout(0);
+        // This point should be reached, even if the value is not used
+        b2Reached = true;
+        yield 'b2';
+        await timers.setTimeout(0);
+        // This point should not be reached
+        b3Reached = true;
+        yield 'b3';
+      } catch (e) {
+        bError = e;
+      } finally {
+        bDone = true;
+      }
+    };
+
+    const merged = mergeIterables([a(), b()]);
+
+    const iter = merged[Symbol.asyncIterator]();
+    expect(await iter.next()).toEqual({ value: 'a', done: false });
+    expect(await iter.next()).toEqual({ value: 'b1', done: false });
+    let error = null;
+    try {
+      await iter.next();
+    } catch (e) {
+      error = e;
+    }
+    expect(error).toEqual(new Error('aError'));
+
+    await timers.setTimeout(2);
+
+    expect(bDone).toEqual(true);
+    expect(bError).toEqual(null);
+    expect(b2Reached).toEqual(true);
+    expect(b3Reached).toEqual(false);
+  });
+
+  test('propagating errors from result', async () => {
+    // We cannot trigger this using a `for await` loop,
+    // only by using the iterator directly.
+    // Skipped since the old iterator doesn't implement this.
+    let aReached = false;
+    let aFinally = false;
+    let aError = null;
+
+    const a = async function* (): AsyncGenerator<number, void> {
+      try {
+        yield 1;
+        aReached = true;
+      } catch (e) {
+        aError = e;
+      } finally {
+        aFinally = true;
+      }
+    };
+
+    const merged = mergeIterables([a()]);
+    const iter = merged[Symbol.asyncIterator]();
+    const r = await iter.next();
+    expect(r).toEqual({ value: 1, done: false });
+    let iterError = null;
+    try {
+      await iter.throw!(new Error('loop error'));
+    } catch (e) {
+      iterError = e;
+    }
+
+    expect(iterError).toEqual(new Error('loop error'));
+    expect(aReached).toEqual(false);
+    expect(aFinally).toEqual(true);
+    expect(aError).toEqual(new Error('loop error'));
+  });
+
+  test('do not block return', async () => {
+    let aDone = false;
+    let bDone = false;
+    let b2Reached = false;
+    let b2Timedout = false;
+    let b3Reached = false;
+
+    const a = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'a';
+      } finally {
+        aDone = true;
+      }
+    };
+    const b = async function* (): AsyncGenerator<string, void> {
+      try {
+        yield 'b1';
+        // This point should be reached, even if the value is not used
+        b2Reached = true;
+        // The final iter.next() should not wait for this to complete
+        await timers.setTimeout(50);
+        b2Timedout = true;
+        yield 'b2';
+        // This point should not be reached
+        b3Reached = true;
+      } finally {
+        bDone = true;
+      }
+    };
+
+    const merged = mergeIterables([a(), b()]);
+
+    const iter = merged[Symbol.asyncIterator]();
+    expect(await iter.next()).toEqual({ value: 'a', done: false });
+    expect(await iter.next()).toEqual({ value: 'b1', done: false });
+    expect(await iter.next()).toEqual({ value: undefined, done: true });
+
+    expect(aDone).toEqual(true);
+    expect(b2Reached).toEqual(true);
+    expect(b2Reached).toEqual(true);
+    expect(b2Timedout).toEqual(false);
+    expect(bDone).toEqual(false);
+
+    await timers.setTimeout(60);
+
+    expect(b2Timedout).toEqual(true);
+    expect(bDone).toEqual(true);
+    expect(b3Reached).toEqual(false);
+  });
+}