@powersync/service-module-mysql 0.7.4 → 0.8.0

package/test/src/BinLogStream.test.ts

@@ -4,7 +4,8 @@ import { ReplicationMetric } from '@powersync/service-types';
 import { v4 as uuid } from 'uuid';
 import { describe, expect, test } from 'vitest';
 import { BinlogStreamTestContext } from './BinlogStreamUtils.js';
-import { describeWithStorage } from './util.js';
+import { createTestDb, describeWithStorage } from './util.js';
+import { qualifiedMySQLTable } from '@module/utils/mysql-utils.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -13,7 +14,7 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
 
-describe('BigLog stream', () => {
+describe('BinLogStream tests', () => {
   describeWithStorage({ timeout: 20_000 }, defineBinlogStreamTests);
 });
 
@@ -34,7 +35,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
     const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
 
-    context.startStreaming();
+    await context.startStreaming();
     const testId = uuid();
     await connectionManager.query(
       `INSERT INTO test_data(id, description, num) VALUES('${testId}', 'test1', 1152921504606846976)`
@@ -48,7 +49,59 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     expect(endTxCount - startTxCount).toEqual(1);
   });
 
-  test('replicating case sensitive table', async () => {
+  test('Replicate multi schema sync rules', async () => {
+    await using context = await BinlogStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(`
+  bucket_definitions:
+    default_schema_test_data:
+      data:
+        - SELECT id, description, num FROM "${connectionManager.databaseName}"."test_data"
+    multi_schema_test_data:
+      data:
+        - SELECT id, description, num FROM "multi_schema"."test_data"
+        `);
+
+    await createTestDb(connectionManager, 'multi_schema');
+
+    await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, num BIGINT)`);
+    const testTable = qualifiedMySQLTable('test_data', 'multi_schema');
+    await connectionManager.query(
+      `CREATE TABLE IF NOT EXISTS ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);`
+    );
+    await context.replicateSnapshot();
+
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+
+    await context.startStreaming();
+
+    const testId = uuid();
+    await connectionManager.query(
+      `INSERT INTO test_data(id, description, num) VALUES('${testId}', 'test1', 1152921504606846976)`
+    );
+
+    const testId2 = uuid();
+    await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${testId2}', 'test2')`);
+
+    const default_data = await context.getBucketData('default_schema_test_data[]');
+    expect(default_data).toMatchObject([
+      putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })
+    ]);
+
+    const multi_schema_data = await context.getBucketData('multi_schema_test_data[]');
+    expect(multi_schema_data).toMatchObject([putOp('test_data', { id: testId2, description: 'test2' })]);
+
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+    expect(endRowCount - startRowCount).toEqual(2);
+    expect(endTxCount - startTxCount).toEqual(2);
+  });
+
+  test('Replicate case sensitive table', async () => {
+    // MySQL inherits the case sensitivity of the underlying OS filesystem.
+    // So Unix-based systems will have case-sensitive tables, but Windows won't.
+    // https://dev.mysql.com/doc/refman/8.4/en/identifier-case-sensitivity.html
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
     await context.updateSyncRules(`
@@ -65,7 +118,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
     const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
 
-    context.startStreaming();
+    await context.startStreaming();
 
     const testId = uuid();
     await connectionManager.query(`INSERT INTO test_DATA(id, description) VALUES('${testId}','test1')`);
@@ -79,50 +132,73 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     expect(endTxCount - startTxCount).toEqual(1);
   });
 
-  // TODO: Not supported yet
-  // test('replicating TRUNCATE', async () => {
-  //   await using context = await BinlogStreamTestContext.create(factory);
-  //   const { connectionManager } = context;
-  //   const syncRuleContent = `
-  // bucket_definitions:
-  //   global:
-  //     data:
-  //       - SELECT id, description FROM "test_data"
-  //   by_test_data:
-  //     parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
-  //     data: []
-  // `;
-  //   await context.updateSyncRules(syncRuleContent);
-  //   await connectionManager.query(`DROP TABLE IF EXISTS test_data`);
-  //   await connectionManager.query(
-  //     `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`
-  //   );
-
-  //   await context.replicateSnapshot();
-  //   context.startStreaming();
-
-  //   const [{ test_id }] = pgwireRows(
-  //     await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
-  //   );
-  //   await connectionManager.query(`TRUNCATE test_data`);
-
-  //   const data = await context.getBucketData('global[]');
-
-  //   expect(data).toMatchObject([
-  //     putOp('test_data', { id: test_id, description: 'test1' }),
-  //     removeOp('test_data', test_id)
-  //   ]);
-  // });
-
-  test('replicating changing primary key', async () => {
+  test('Replicate matched wild card tables in sync rules', async () => {
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
+    await context.updateSyncRules(`
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM "test_data_%"`);
+
+    await connectionManager.query(`CREATE TABLE test_data_1 (id CHAR(36) PRIMARY KEY, description TEXT)`);
+    await connectionManager.query(`CREATE TABLE test_data_2 (id CHAR(36) PRIMARY KEY, description TEXT)`);
+
+    const testId11 = uuid();
+    await connectionManager.query(`INSERT INTO test_data_1(id, description) VALUES('${testId11}','test11')`);
+
+    const testId21 = uuid();
+    await connectionManager.query(`INSERT INTO test_data_2(id, description) VALUES('${testId21}','test21')`);
+
+    await context.replicateSnapshot();
+    await context.startStreaming();
+
+    const testId12 = uuid();
+    await connectionManager.query(`INSERT INTO test_data_1(id, description) VALUES('${testId12}', 'test12')`);
+
+    const testId22 = uuid();
+    await connectionManager.query(`INSERT INTO test_data_2(id, description) VALUES('${testId22}', 'test22')`);
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([
+      putOp('test_data_1', { id: testId11, description: 'test11' }),
+      putOp('test_data_2', { id: testId21, description: 'test21' }),
+      putOp('test_data_1', { id: testId12, description: 'test12' }),
+      putOp('test_data_2', { id: testId22, description: 'test22' })
+    ]);
+  });
+
+  test('Handle table TRUNCATE events', async () => {
+    await using context = await BinlogStreamTestContext.open(factory);
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    const { connectionManager } = context;
+    await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`);
+
+    await context.replicateSnapshot();
+    await context.startStreaming();
+
+    const testId = uuid();
+    await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`);
+    await connectionManager.query(`TRUNCATE TABLE test_data`);
+
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([
+      putOp('test_data', { id: testId, description: 'test1' }),
+      removeOp('test_data', testId)
+    ]);
+  });
+
+  test('Handle changes in a replicated table primary key', async () => {
+    await using context = await BinlogStreamTestContext.open(factory);
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
+    const { connectionManager } = context;
     await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`);
 
     await context.replicateSnapshot();
-    context.startStreaming();
+    await context.startStreaming();
 
     const testId1 = uuid();
     await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}','test1')`);
@@ -154,7 +230,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     ]);
   });
 
-  test('initial sync', async () => {
+  test('Initial snapshot sync', async () => {
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
@@ -167,7 +243,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
 
     await context.replicateSnapshot();
-    context.startStreaming();
+    await context.startStreaming();
 
     const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
     const data = await context.getBucketData('global[]');
@@ -175,7 +251,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     expect(endRowCount - startRowCount).toEqual(1);
   });
 
-  test('snapshot with date values', async () => {
+  test('Snapshot with date values', async () => {
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
     await context.updateSyncRules(`
@@ -195,7 +271,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
       `);
 
     await context.replicateSnapshot();
-    context.startStreaming();
+    await context.startStreaming();
 
     const data = await context.getBucketData('global[]');
     expect(data).toMatchObject([
@@ -209,7 +285,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     ]);
   });
 
-  test('replication with date values', async () => {
+  test('Replication with date values', async () => {
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
     await context.updateSyncRules(`
@@ -228,7 +304,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
     const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
 
-    context.startStreaming();
+    await context.startStreaming();
 
     const testId = uuid();
     await connectionManager.query(`
@@ -259,7 +335,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     expect(endTxCount - startTxCount).toEqual(2);
   });
 
-  test('table not in sync rules', async () => {
+  test('Replication for tables not in the sync rules are ignored', async () => {
     await using context = await BinlogStreamTestContext.open(factory);
     const { connectionManager } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
@@ -271,7 +347,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
     const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
     const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
 
-    context.startStreaming();
+    await context.startStreaming();
 
     await connectionManager.query(`INSERT INTO test_donotsync(id, description) VALUES('${uuid()}','test1')`);
     const data = await context.getBucketData('global[]');
@@ -300,7 +376,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
       await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, num BIGINT)`);
 
       await context.replicateSnapshot();
-      context.startStreaming();
+      await context.startStreaming();
       await connectionManager.query(
         `INSERT INTO test_data(id, description, num) VALUES('${testId1}', 'test1', 1152921504606846976)`
       );
@@ -315,7 +391,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) {
       await context.loadActiveSyncRules();
       // Does not actually do a snapshot again - just does the required intialization.
       await context.replicateSnapshot();
-      context.startStreaming();
+      await context.startStreaming();
       await connectionManager.query(`INSERT INTO test_data(id, description, num) VALUES('${testId2}', 'test2', 0)`);
       const data = await context.getBucketData('global[]');
 

package/test/src/BinlogStreamUtils.ts

@@ -16,6 +16,7 @@ import {
 import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import mysqlPromise from 'mysql2/promise';
 import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
+import timers from 'timers/promises';
 
 /**
  * Tests operating on the binlog stream need to configure the stream and manage asynchronous
@@ -112,15 +113,24 @@ export class BinlogStreamTestContext {
 
   async replicateSnapshot() {
     await this.binlogStream.initReplication();
-    await this.storage!.autoActivate();
     this.replicationDone = true;
   }
 
-  startStreaming() {
+  async startStreaming() {
     if (!this.replicationDone) {
       throw new Error('Call replicateSnapshot() before startStreaming()');
     }
     this.streamPromise = this.binlogStream.streamChanges();
+
+    // Wait for the replication to start before returning.
+    // This avoids a bunch of unpredictable race conditions that appear in testing
+    return new Promise<void>(async (resolve) => {
+      while (this.binlogStream.isStartingReplication) {
+        await timers.setTimeout(50);
+      }
+
+      resolve();
+    });
   }
 
   async getCheckpoint(options?: { timeout?: number }): Promise<InternalOpId> {

package/test/src/parser-utils.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from 'vitest';
+import { matchedSchemaChangeQuery } from '@module/utils/parser-utils.js';
+
+describe('MySQL Parser Util Tests', () => {
+  test('matchedSchemaChangeQuery function', () => {
+    const matcher = (tableName: string) => tableName === 'users';
+
+    // DDL matches and table name matches
+    expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', [matcher])).toBeTruthy();
+    expect(matchedSchemaChangeQuery('DROP TABLE users', [matcher])).toBeTruthy();
+    expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', [matcher])).toBeTruthy();
+    expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', [matcher])).toBeTruthy();
+
+    // Can handle backticks in table names
+    expect(
+      matchedSchemaChangeQuery('ALTER TABLE `clientSchema`.`users` ADD COLUMN name VARCHAR(255)', [matcher])
+    ).toBeTruthy();
+
+    // DDL matches, but table name does not match
+    expect(matchedSchemaChangeQuery('DROP TABLE clientSchema.clients', [matcher])).toBeFalsy();
+    // No DDL match
+    expect(matchedSchemaChangeQuery('SELECT * FROM users', [matcher])).toBeFalsy();
+  });
+});