@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/test/src/slow_tests.test.ts

@@ -1,296 +0,0 @@
-import * as bson from 'bson';
-import * as mongo from 'mongodb';
-import { afterEach, describe, expect, test } from 'vitest';
-import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
-import { getClientCheckpoint } from '../../src/util/utils.js';
-import { env } from './env.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory, TEST_CONNECTION_OPTIONS, connectPgPool } from './util.js';
-
-import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteRow } from '@powersync/service-sync-rules';
-import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
-import { PgManager } from '../../src/util/PgManager.js';
-
-describe('slow tests - mongodb', function () {
-  // These are slow, inconsistent tests.
-  // Not run on every test run, but we do run on CI, or when manually debugging issues.
-  if (env.CI || env.SLOW_TESTS) {
-    defineSlowTests(MONGO_STORAGE_FACTORY);
-  } else {
-    // Need something in this file.
-    test('no-op', () => {});
-  }
-});
-
-function defineSlowTests(factory: StorageFactory) {
-  let walStream: WalStream | undefined;
-  let connections: PgManager | undefined;
-  let abortController: AbortController | undefined;
-  let streamPromise: Promise<void> | undefined;
-
-  afterEach(async () => {
-    // This cleans up, similar to WalStreamTestContext.dispose().
-    // These tests are a little more complex than what is supported by WalStreamTestContext.
-    abortController?.abort();
-    await streamPromise;
-    streamPromise = undefined;
-    connections?.destroy();
-
-    connections = undefined;
-    walStream = undefined;
-    abortController = undefined;
-  });
-
-  const TEST_DURATION_MS = 15_000;
-  const TIMEOUT_MARGIN_MS = env.CI ? 30_000 : 15_000;
-
-  // Test repeatedly replicating inserts and deletes, then check that we get
-  // consistent data out at the end.
-  //
-  // Past issues that this could reproduce intermittently:
-  // * Skipping LSNs after a keepalive message
-  // * Skipping LSNs when source transactions overlap
-  test(
-    'repeated replication',
-    async () => {
-      const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-      const replicationConnection = await connections.replicationConnection();
-      const pool = connections.pool;
-      const f = (await factory()) as MongoBucketStorage;
-
-      const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT * FROM "test_data"
-`;
-      const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-      const storage = f.getInstance(syncRules.parsed());
-      abortController = new AbortController();
-      const options: WalStreamOptions = {
-        abort_signal: abortController.signal,
-        connections,
-        storage: storage,
-        factory: f
-      };
-      walStream = new WalStream(options);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
-      );
-      await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
-
-      await walStream.initReplication(replicationConnection);
-      await storage.autoActivate();
-      let abort = false;
-      streamPromise = walStream.streamChanges(replicationConnection).finally(() => {
-        abort = true;
-      });
-      const start = Date.now();
-
-      while (!abort && Date.now() - start < TEST_DURATION_MS) {
-        const bg = async () => {
-          for (let j = 0; j < 1 && !abort; j++) {
-            const n = 1;
-            let statements: pgwire.Statement[] = [];
-            for (let i = 0; i < n; i++) {
-              const description = `test${i}`;
-              statements.push({
-                statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
-                params: [
-                  { type: 'varchar', value: description },
-                  { type: 'float8', value: Math.random() }
-                ]
-              });
-            }
-            const results = await pool.query(...statements);
-            const ids = results.results.map((sub) => {
-              return sub.rows[0][0] as string;
-            });
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
-
-            if (Math.random() > 0.5) {
-              const updateStatements: pgwire.Statement[] = ids.map((id) => {
-                return {
-                  statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
-                  params: [
-                    { type: 'uuid', value: id },
-                    { type: 'float8', value: Math.random() }
-                  ]
-                };
-              });
-
-              await pool.query(...updateStatements);
-              if (Math.random() > 0.5) {
-                // Special case - an update that doesn't change data
-                await pool.query(...updateStatements);
-              }
-            }
-
-            const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-              return {
-                statement: `DELETE FROM test_data WHERE id = $1`,
-                params: [{ type: 'uuid', value: id }]
-              };
-            });
-            await pool.query(...deleteStatements);
-
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
-          }
-        };
-
-        // Call the above loop multiple times concurrently
-        let promises = [1, 2, 3].map((i) => bg());
-        await Promise.all(promises);
-
-        // Wait for replication to finish
-        let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS });
-
-        // Check that all inserts have been deleted again
-        const docs = await f.db.current_data.find().toArray();
-        const transformed = docs.map((doc) => {
-          return bson.deserialize((doc.data as mongo.Binary).buffer) as SqliteRow;
-        });
-        expect(transformed).toEqual([]);
-
-        // Check that each PUT has a REMOVE
-        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
-        let active = new Set<string>();
-        for (let op of ops) {
-          const key = op.source_key.toHexString();
-          if (op.op == 'PUT') {
-            active.add(key);
-          } else if (op.op == 'REMOVE') {
-            active.delete(key);
-          }
-        }
-        if (active.size > 0) {
-          throw new Error(`${active.size} rows not removed`);
-        }
-      }
-
-      abortController.abort();
-      await streamPromise;
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
-
-  // Test repeatedly performing initial replication.
-  //
-  // If the first LSN does not correctly match with the first replication transaction,
-  // we may miss some updates.
-  test(
-    'repeated initial replication',
-    async () => {
-      const pool = await connectPgPool();
-      const f = await factory();
-
-      const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-`;
-      const syncRules = await f.updateSyncRules({ content: syncRuleContent });
-      const storage = f.getInstance(syncRules.parsed());
-
-      // 1. Setup some base data that will be replicated in initial replication
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      let statements: pgwire.Statement[] = [];
-
-      const n = Math.floor(Math.random() * 200);
-      for (let i = 0; i < n; i++) {
-        statements.push({
-          statement: `INSERT INTO test_data(description) VALUES('test_init')`
-        });
-      }
-      await pool.query(...statements);
-
-      const start = Date.now();
-      let i = 0;
-
-      while (Date.now() - start < TEST_DURATION_MS) {
-        // 2. Each iteration starts with a clean slate
-        await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`);
-        i += 1;
-
-        const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
-        const replicationConnection = await connections.replicationConnection();
-
-        abortController = new AbortController();
-        const options: WalStreamOptions = {
-          abort_signal: abortController.signal,
-          connections,
-          storage: storage,
-          factory: f
-        };
-        walStream = new WalStream(options);
-
-        await storage.clear();
-
-        // 3. Start initial replication, then streaming, but don't wait for any of this
-        let initialReplicationDone = false;
-        streamPromise = (async () => {
-          await walStream.initReplication(replicationConnection);
-          await storage.autoActivate();
-          initialReplicationDone = true;
-          await walStream.streamChanges(replicationConnection);
-        })()
-          .catch((e) => {
-            initialReplicationDone = true;
-            throw e;
-          })
-          .then((v) => {
-            return v;
-          });
-
-        // 4. While initial replication is still running, write more changes
-        while (!initialReplicationDone) {
-          let statements: pgwire.Statement[] = [];
-          const n = Math.floor(Math.random() * 10) + 1;
-          for (let i = 0; i < n; i++) {
-            const description = `test${i}`;
-            statements.push({
-              statement: `INSERT INTO test_data(description) VALUES('test1') returning id as test_id`,
-              params: [{ type: 'varchar', value: description }]
-            });
-          }
-          const results = await pool.query(...statements);
-          const ids = results.results.map((sub) => {
-            return sub.rows[0][0] as string;
-          });
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
-          const deleteStatements: pgwire.Statement[] = ids.map((id) => {
-            return {
-              statement: `DELETE FROM test_data WHERE id = $1`,
-              params: [{ type: 'uuid', value: id }]
-            };
-          });
-          await pool.query(...deleteStatements);
-          await new Promise((resolve) => setTimeout(resolve, Math.random() * 10));
-        }
-
-        // 5. Once initial replication is done, wait for the streaming changes to complete syncing.
-        // getClientCheckpoint() effectively waits for the above replication to complete
-        // Race with streamingPromise to catch replication errors here.
-        let checkpoint = await Promise.race([
-          getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
-          streamPromise
-        ]);
-        if (typeof checkpoint == undefined) {
-          // This indicates an issue with the test setup - streamingPromise completed instead
-          // of getClientCheckpoint()
-          throw new Error('Test failure - streamingPromise completed');
-        }
-
-        abortController.abort();
-        await streamPromise;
-        await connections.end();
-      }
-    },
-    { timeout: TEST_DURATION_MS + TIMEOUT_MARGIN_MS }
-  );
-}

package/test/src/validation.test.ts

@@ -1,63 +0,0 @@
-import { expect, test } from 'vitest';
-import { MONGO_STORAGE_FACTORY } from './util.js';
-import { walStreamTest } from './wal_stream_utils.js';
-import { WalConnection } from '../../src/replication/WalConnection.js';
-
-// Not quite a walStreamTest, but it helps to manage the connection
-test(
-  'validate tables',
-  walStreamTest(MONGO_STORAGE_FACTORY, async (context) => {
-    const { pool } = context;
-
-    await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-    const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-      - SELECT * FROM "other"
-      - SELECT * FROM "other%"
-`;
-
-    const syncRules = await context.factory.updateSyncRules({ content: syncRuleContent });
-
-    const walConnection = new WalConnection({
-      db: pool,
-      sync_rules: syncRules.parsed().sync_rules
-    });
-
-    const tablePatterns = syncRules.parsed().sync_rules.getSourceTables();
-    const tableInfo = await walConnection.getDebugTablesInfo(tablePatterns);
-    expect(tableInfo).toEqual([
-      {
-        schema: 'public',
-        pattern: 'test_data',
-        wildcard: false,
-        table: {
-          schema: 'public',
-          name: 'test_data',
-          replication_id: ['id'],
-          pattern: undefined,
-          data_queries: true,
-          parameter_queries: false,
-          errors: []
-        }
-      },
-      {
-        schema: 'public',
-        pattern: 'other',
-        wildcard: false,
-        table: {
-          schema: 'public',
-          name: 'other',
-          replication_id: [],
-          data_queries: true,
-          parameter_queries: false,
-          errors: [{ level: 'warning', message: 'Table "public"."other" not found.' }]
-        }
-      },
-      { schema: 'public', pattern: 'other%', wildcard: true, tables: [] }
-    ]);
-  })
-);

package/test/src/wal_stream.test.ts

@@ -1,314 +0,0 @@
-import * as crypto from 'crypto';
-import { describe, expect, test } from 'vitest';
-import { BucketStorageFactory } from '@/storage/BucketStorage.js';
-import { MONGO_STORAGE_FACTORY } from './util.js';
-import { putOp, removeOp, walStreamTest } from './wal_stream_utils.js';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { Metrics } from '@/metrics/Metrics.js';
-import { container } from '@powersync/lib-services-framework';
-
-type StorageFactory = () => Promise<BucketStorageFactory>;
-
-const BASIC_SYNC_RULES = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-`;
-
-describe(
-  'wal stream - mongodb',
-  function () {
-    defineWalStreamTests(MONGO_STORAGE_FACTORY);
-  },
-  { timeout: 20_000 }
-);
-
-function defineWalStreamTests(factory: StorageFactory) {
-  test(
-    'replicating basic values',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description, num FROM "test_data"`);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num int8)`
-      );
-
-      await context.replicateSnapshot();
-
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(
-          `INSERT INTO test_data(description, num) VALUES('test1', 1152921504606846976) returning id as test_id`
-        )
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([
-        putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })
-      ]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-      expect(endRowCount - startRowCount).toEqual(1);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-
-  test(
-    'replicating case sensitive table',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-      bucket_definitions:
-        global:
-          data:
-            - SELECT id, description FROM "test_DATA"
-      `);
-
-      await pool.query(`DROP TABLE IF EXISTS "test_DATA"`);
-      await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-      expect(endRowCount - startRowCount).toEqual(1);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-
-  test(
-    'replicating TOAST values',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(`
-      bucket_definitions:
-        global:
-          data:
-            - SELECT id, name, description FROM "test_data"
-      `);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(
-        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)`
-      );
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      // Must be > 8kb after compression
-      const largeDescription = crypto.randomBytes(20_000).toString('hex');
-      const [{ test_id }] = pgwireRows(
-        await pool.query({
-          statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`,
-          params: [{ type: 'varchar', value: largeDescription }]
-        })
-      );
-
-      await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`);
-
-      const data = await context.getBucketData('global[]');
-      expect(data.slice(0, 1)).toMatchObject([
-        putOp('test_data', { id: test_id, name: 'test1', description: largeDescription })
-      ]);
-      expect(data.slice(1)).toMatchObject([
-        putOp('test_data', { id: test_id, name: 'test2', description: largeDescription })
-      ]);
-    })
-  );
-
-  test(
-    'replicating TRUNCATE',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      const syncRuleContent = `
-bucket_definitions:
-  global:
-    data:
-      - SELECT id, description FROM "test_data"
-  by_test_data:
-    parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
-    data: []
-`;
-      await context.updateSyncRules(syncRuleContent);
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-      await pool.query(`TRUNCATE test_data`);
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([
-        putOp('test_data', { id: test_id, description: 'test1' }),
-        removeOp('test_data', test_id)
-      ]);
-    })
-  );
-
-  test(
-    'replicating changing primary key',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const [{ test_id: test_id2 }] = pgwireRows(
-        await pool.query(
-          `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id`
-        )
-      );
-
-      // This update may fail replicating with:
-      // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
-      await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`);
-
-      // Re-use old id again
-      await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`);
-      await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`);
-
-      const data = await context.getBucketData('global[]');
-      expect(data).toMatchObject([
-        // Initial insert
-        putOp('test_data', { id: test_id, description: 'test1' }),
-        // Update id, then description
-        removeOp('test_data', test_id),
-        putOp('test_data', { id: test_id2, description: 'test2a' }),
-        putOp('test_data', { id: test_id2, description: 'test2b' }),
-        // Re-use old id
-        putOp('test_data', { id: test_id, description: 'test1b' }),
-        putOp('test_data', { id: test_id, description: 'test1c' })
-      ]);
-    })
-  );
-
-  test(
-    'initial sync',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-
-      await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-      );
-
-      await context.replicateSnapshot();
-      context.startStreaming();
-
-      const data = await context.getBucketData('global[]');
-      expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]);
-    })
-  );
-
-  test(
-    'record too large',
-    walStreamTest(factory, async (context) => {
-      await context.updateSyncRules(`bucket_definitions:
-      global:
-        data:
-          - SELECT id, description, other FROM "test_data"`);
-      const { pool } = context;
-
-      await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
-
-      await context.replicateSnapshot();
-
-      // 4MB
-      const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
-      // 18MB
-      const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
-
-      await pool.query({
-        statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
-        params: [{ type: 'varchar', value: tooLargeDescription }]
-      });
-      await pool.query({
-        statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
-        params: [{ type: 'varchar', value: largeDescription }]
-      });
-
-      context.startStreaming();
-
-      const data = await context.getBucketData('global[]');
-      expect(data.length).toEqual(1);
-      const row = JSON.parse(data[0].data as string);
-      delete row.description;
-      expect(row).toEqual({ id: 't1', other: 'foo' });
-      delete data[0].data;
-      expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
-    })
-  );
-
-  test(
-    'table not in sync rules',
-    walStreamTest(factory, async (context) => {
-      const { pool } = context;
-      await context.updateSyncRules(BASIC_SYNC_RULES);
-
-      await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`);
-
-      await context.replicateSnapshot();
-
-      const metrics = container.getImplementation(Metrics);
-      const startRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const startTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      context.startStreaming();
-
-      const [{ test_id }] = pgwireRows(
-        await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`)
-      );
-
-      const data = await context.getBucketData('global[]');
-
-      expect(data).toMatchObject([]);
-      const endRowCount = (await metrics.getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
-      const endTxCount = (await metrics.getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
-
-      // There was a transaction, but we should not replicate any actual data
-      expect(endRowCount - startRowCount).toEqual(0);
-      expect(endTxCount - startTxCount).toEqual(1);
-    })
-  );
-}