@powersync/service-module-mysql 0.8.0 → 0.9.0

package/test/src/BinLogListener.test.ts

@@ -1,22 +1,15 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
-import {
-  BinLogEventHandler,
-  BinLogListener,
-  Row,
-  SchemaChange,
-  SchemaChangeType
-} from '@module/replication/zongji/BinLogListener.js';
+import { BinLogListener, SchemaChange, SchemaChangeType } from '@module/replication/zongji/BinLogListener.js';
 import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
-import { clearTestDb, createTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
-import { v4 as uuid } from 'uuid';
-import * as common from '@module/common/common-index.js';
 import {
-  createRandomServerId,
-  getMySQLVersion,
-  qualifiedMySQLTable,
-  satisfiesVersion
-} from '@module/utils/mysql-utils.js';
-import { TableMapEntry } from '@powersync/mysql-zongji';
+  clearTestDb,
+  createBinlogListener,
+  createTestDb,
+  TEST_CONNECTION_OPTIONS,
+  TestBinLogEventHandler
+} from './util.js';
+import { v4 as uuid } from 'uuid';
+import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js';
 import crypto from 'crypto';
 import { TablePattern } from '@powersync/service-sync-rules';
 
@@ -46,7 +39,11 @@ describe('BinlogListener tests', () => {
     await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
     connection.release();
     eventHandler = new TestBinLogEventHandler();
-    binLogListener = await createBinlogListener();
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      sourceTables: [new TablePattern(connectionManager.databaseName, 'test_DATA')],
+      eventHandler
+    });
   });
 
   afterAll(async () => {
@@ -106,6 +103,14 @@ describe('BinlogListener tests', () => {
     await binLogListener.stop();
   });
 
+  test('Keepalive event', async () => {
+    binLogListener.options.keepAliveInactivitySeconds = 1;
+    await binLogListener.start();
+    await vi.waitFor(() => expect(eventHandler.lastKeepAlive).toBeDefined(), { timeout: 10000 });
+    await binLogListener.stop();
+    expect(eventHandler.lastKeepAlive).toEqual(binLogListener.options.startGTID.comparable);
+  });
+
   test('Schema change event: Rename table', async () => {
     await binLogListener.start();
     await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`);
@@ -276,7 +281,11 @@ describe('BinlogListener tests', () => {
   test('Schema change event: Drop and Add primary key', async () => {
     await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
     await binLogListener.start();
     await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`);
     await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`);
@@ -301,7 +310,11 @@ describe('BinlogListener tests', () => {
   test('Schema change event: Add and drop unique constraint', async () => {
     await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
     await binLogListener.start();
     await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`);
     await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`);
@@ -326,7 +339,11 @@ describe('BinlogListener tests', () => {
   test('Schema change event: Add and drop a unique index', async () => {
     await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
     await binLogListener.start();
     await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`);
     await connectionManager.query(`DROP INDEX description_idx ON test_constraints`);
@@ -367,7 +384,11 @@ describe('BinlogListener tests', () => {
     // If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once
     // all the schema changes have been processed
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_multiple')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
 
     await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`);
     await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`);
@@ -388,7 +409,11 @@ describe('BinlogListener tests', () => {
   test('Unprocessed binlog event received that does match the current table schema', async () => {
     // If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
 
     await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
     await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
@@ -403,7 +428,11 @@ describe('BinlogListener tests', () => {
 
   test('Unprocessed binlog event received for a dropped table', async () => {
     const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
 
     // If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error
     await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
@@ -424,7 +453,11 @@ describe('BinlogListener tests', () => {
       new TablePattern(connectionManager.databaseName, 'test_DATA'),
       new TablePattern('multi_schema', 'test_DATA_multi')
     ];
-    binLogListener = await createBinlogListener(sourceTables);
+    binLogListener = await createBinlogListener({
+      connectionManager,
+      eventHandler,
+      sourceTables
+    });
     await binLogListener.start();
 
     // Default database insert into test_DATA
@@ -439,28 +472,6 @@ describe('BinlogListener tests', () => {
     assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi');
   });
 
-  async function createBinlogListener(
-    sourceTables?: TablePattern[],
-    startPosition?: common.BinLogPosition
-  ): Promise<BinLogListener> {
-    if (!sourceTables) {
-      sourceTables = [new TablePattern(connectionManager.databaseName, 'test_DATA')];
-    }
-
-    if (!startPosition) {
-      const fromGTID = await getFromGTID(connectionManager);
-      startPosition = fromGTID.position;
-    }
-
-    return new BinLogListener({
-      connectionManager: connectionManager,
-      eventHandler: eventHandler,
-      startPosition: startPosition,
-      sourceTables: sourceTables,
-      serverId: createRandomServerId(1)
-    });
-  }
-
   function assertSchemaChange(
     change: SchemaChange,
     type: SchemaChangeType,
@@ -477,14 +488,6 @@ describe('BinlogListener tests', () => {
   }
 });
 
-async function getFromGTID(connectionManager: MySQLConnectionManager) {
-  const connection = await connectionManager.getConnection();
-  const fromGTID = await common.readExecutedGtid(connection);
-  connection.release();
-
-  return fromGTID;
-}
-
 async function insertRows(connectionManager: MySQLConnectionManager, count: number) {
   for (let i = 0; i < count; i++) {
     await connectionManager.query(
@@ -500,45 +503,3 @@ async function updateRows(connectionManager: MySQLConnectionManager) {
 async function deleteRows(connectionManager: MySQLConnectionManager) {
   await connectionManager.query(`DELETE FROM test_DATA`);
 }
-
-class TestBinLogEventHandler implements BinLogEventHandler {
-  rowsWritten = 0;
-  rowsUpdated = 0;
-  rowsDeleted = 0;
-  commitCount = 0;
-  schemaChanges: SchemaChange[] = [];
-
-  unpause: ((value: void | PromiseLike<void>) => void) | undefined;
-  private pausedPromise: Promise<void> | undefined;
-
-  pause() {
-    this.pausedPromise = new Promise((resolve) => {
-      this.unpause = resolve;
-    });
-  }
-
-  async onWrite(rows: Row[], tableMap: TableMapEntry) {
-    if (this.pausedPromise) {
-      await this.pausedPromise;
-    }
-    this.rowsWritten = this.rowsWritten + rows.length;
-  }
-
-  async onUpdate(afterRows: Row[], beforeRows: Row[], tableMap: TableMapEntry) {
-    this.rowsUpdated = this.rowsUpdated + afterRows.length;
-  }
-
-  async onDelete(rows: Row[], tableMap: TableMapEntry) {
-    this.rowsDeleted = this.rowsDeleted + rows.length;
-  }
-
-  async onCommit(lsn: string) {
-    this.commitCount++;
-  }
-
-  async onSchemaChange(change: SchemaChange) {
-    this.schemaChanges.push(change);
-  }
-  async onTransactionStart(options: { timestamp: Date }) {}
-  async onRotate() {}
-}

package/test/src/mysql-to-sqlite.test.ts

@@ -1,4 +1,4 @@
-import { SqliteRow } from '@powersync/service-sync-rules';
+import { SqliteInputRow, SqliteRow } from '@powersync/service-sync-rules';
 import { afterAll, describe, expect, test } from 'vitest';
 import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js';
@@ -298,7 +298,7 @@ INSERT INTO test_data (
   });
 });
 
-async function getDatabaseRows(connection: MySQLConnectionManager, tableName: string): Promise<SqliteRow[]> {
+async function getDatabaseRows(connection: MySQLConnectionManager, tableName: string): Promise<SqliteInputRow[]> {
   const [results, fields] = await connection.query(`SELECT * FROM ${tableName}`);
   const columns = toColumnDescriptors(fields);
   return results.map((row) => common.toSQLiteRow(row, columns));
@@ -307,15 +307,15 @@ async function getDatabaseRows(connection: MySQLConnectionManager, tableName: st
 /**
  * Return all the inserts from the first transaction in the binlog stream.
  */
-async function getReplicatedRows(expectedTransactionsCount?: number): Promise<SqliteRow[]> {
-  let transformed: SqliteRow[] = [];
+async function getReplicatedRows(expectedTransactionsCount?: number): Promise<SqliteInputRow[]> {
+  let transformed: SqliteInputRow[] = [];
   const zongji = new ZongJi({
     host: TEST_CONNECTION_OPTIONS.hostname,
     user: TEST_CONNECTION_OPTIONS.username,
     password: TEST_CONNECTION_OPTIONS.password
   });
 
-  const completionPromise = new Promise<SqliteRow[]>((resolve, reject) => {
+  const completionPromise = new Promise<SqliteInputRow[]>((resolve, reject) => {
     zongji.on('binlog', (evt: BinLogEvent) => {
       try {
         if (eventIsWriteMutation(evt)) {

package/test/src/schema-changes.test.ts

@@ -652,12 +652,8 @@ function defineTests(factory: storage.TestStorageFactory) {
     await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t3','test3')`);
     await connectionManager.query(`DROP TABLE ${testTable}`);
 
-    // Force a commit on the watched schema to advance the checkpoint
-    await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`);
-
     const data = await context.getBucketData('global[]');
 
-    // Should only include the entry used to advance the checkpoint
-    expect(data).toMatchObject([PUT_T1]);
+    expect(data).toMatchObject([]);
   });
 }

package/test/src/util.ts

@@ -1,5 +1,5 @@
 import * as types from '@module/types/types.js';
-import { getMySQLVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js';
+import { createRandomServerId, getMySQLVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import mysqlPromise from 'mysql2/promise';
@@ -7,6 +7,10 @@ import { env } from './env.js';
 import { describe, TestOptions } from 'vitest';
 import { TestStorageFactory } from '@powersync/service-core';
 import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import { BinLogEventHandler, BinLogListener, Row, SchemaChange } from '@module/replication/zongji/BinLogListener.js';
+import { TableMapEntry } from '@powersync/mysql-zongji';
+import * as common from '@module/common/common-index.js';
+import { TablePattern } from '@powersync/service-sync-rules';
 
 export const TEST_URI = env.MYSQL_TEST_URI;
 
@@ -58,3 +62,79 @@ export async function createTestDb(connectionManager: MySQLConnectionManager, db
   await connectionManager.query(`DROP DATABASE IF EXISTS ${dbName}`);
   await connectionManager.query(`CREATE DATABASE IF NOT EXISTS ${dbName}`);
 }
+
+export async function getFromGTID(connectionManager: MySQLConnectionManager) {
+  const connection = await connectionManager.getConnection();
+  const fromGTID = await common.readExecutedGtid(connection);
+  connection.release();
+
+  return fromGTID;
+}
+
+export interface CreateBinlogListenerParams {
+  connectionManager: MySQLConnectionManager;
+  eventHandler: BinLogEventHandler;
+  sourceTables: TablePattern[];
+  startGTID?: common.ReplicatedGTID;
+}
+export async function createBinlogListener(params: CreateBinlogListenerParams): Promise<BinLogListener> {
+  let { connectionManager, eventHandler, sourceTables, startGTID } = params;
+
+  if (!startGTID) {
+    startGTID = await getFromGTID(connectionManager);
+  }
+
+  return new BinLogListener({
+    connectionManager: connectionManager,
+    eventHandler: eventHandler,
+    startGTID: startGTID!,
+    sourceTables: sourceTables,
+    serverId: createRandomServerId(1)
+  });
+}
+
+export class TestBinLogEventHandler implements BinLogEventHandler {
+  rowsWritten = 0;
+  rowsUpdated = 0;
+  rowsDeleted = 0;
+  commitCount = 0;
+  schemaChanges: SchemaChange[] = [];
+  lastKeepAlive: string | undefined;
+
+  unpause: ((value: void | PromiseLike<void>) => void) | undefined;
+  private pausedPromise: Promise<void> | undefined;
+
+  pause() {
+    this.pausedPromise = new Promise((resolve) => {
+      this.unpause = resolve;
+    });
+  }
+
+  async onWrite(rows: Row[], tableMap: TableMapEntry) {
+    if (this.pausedPromise) {
+      await this.pausedPromise;
+    }
+    this.rowsWritten = this.rowsWritten + rows.length;
+  }
+
+  async onUpdate(afterRows: Row[], beforeRows: Row[], tableMap: TableMapEntry) {
+    this.rowsUpdated = this.rowsUpdated + afterRows.length;
+  }
+
+  async onDelete(rows: Row[], tableMap: TableMapEntry) {
+    this.rowsDeleted = this.rowsDeleted + rows.length;
+  }
+
+  async onCommit(lsn: string) {
+    this.commitCount++;
+  }
+
+  async onSchemaChange(change: SchemaChange) {
+    this.schemaChanges.push(change);
+  }
+  async onTransactionStart(options: { timestamp: Date }) {}
+  async onRotate() {}
+  async onKeepAlive(lsn: string) {
+    this.lastKeepAlive = lsn;
+  }
+}