@powersync/service-module-mysql 0.6.5 → 0.7.1

package/src/replication/zongji/BinLogListener.ts

@@ -0,0 +1,243 @@
+import * as common from '../../common/common-index.js';
+import async from 'async';
+import { BinLogEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji';
+import * as zongji_utils from './zongji-utils.js';
+import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { MySQLConnectionManager } from '../MySQLConnectionManager.js';
+
+// Maximum time the processing queue can be paused before resuming automatically
+// MySQL server will automatically terminate replication connections after 60 seconds of inactivity, so this guards against that.
+const MAX_QUEUE_PAUSE_TIME_MS = 45_000;
+
+export type Row = Record<string, any>;
+
+export interface BinLogEventHandler {
+  onTransactionStart: (options: { timestamp: Date }) => Promise<void>;
+  onRotate: () => Promise<void>;
+  onWrite: (rows: Row[], tableMap: TableMapEntry) => Promise<void>;
+  onUpdate: (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => Promise<void>;
+  onDelete: (rows: Row[], tableMap: TableMapEntry) => Promise<void>;
+  onCommit: (lsn: string) => Promise<void>;
+}
+
+export interface BinLogListenerOptions {
+  connectionManager: MySQLConnectionManager;
+  eventHandler: BinLogEventHandler;
+  includedTables: string[];
+  serverId: number;
+  startPosition: common.BinLogPosition;
+  logger?: Logger;
+}
+
+/**
+ *  Wrapper class for the Zongji BinLog listener. Internally handles the creation and management of the listener and posts
+ *  events on the provided BinLogEventHandler.
+ */
+export class BinLogListener {
+  private connectionManager: MySQLConnectionManager;
+  private eventHandler: BinLogEventHandler;
+  private binLogPosition: common.BinLogPosition;
+  private currentGTID: common.ReplicatedGTID | null;
+  private logger: Logger;
+
+  zongji: ZongJi;
+  processingQueue: async.QueueObject<BinLogEvent>;
+  /**
+   *  The combined size in bytes of all the binlog events currently in the processing queue.
+   */
+  queueMemoryUsage: number;
+
+  constructor(public options: BinLogListenerOptions) {
+    this.logger = options.logger ?? defaultLogger;
+    this.connectionManager = options.connectionManager;
+    this.eventHandler = options.eventHandler;
+    this.binLogPosition = options.startPosition;
+    this.currentGTID = null;
+
+    this.processingQueue = async.queue(this.createQueueWorker(), 1);
+    this.queueMemoryUsage = 0;
+    this.zongji = this.createZongjiListener();
+  }
+
+  /**
+   *  The queue memory limit in bytes as defined in the connection options.
+   *  @private
+   */
+  private get queueMemoryLimit(): number {
+    return this.connectionManager.options.binlog_queue_memory_limit * 1024 * 1024;
+  }
+
+  public async start(): Promise<void> {
+    if (this.isStopped) {
+      return;
+    }
+    this.logger.info(`Starting replication. Created replica client with serverId:${this.options.serverId}`);
+
+    this.zongji.start({
+      // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive
+      // tablemap events always need to be included for the other row events to work
+      includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
+      includeSchema: { [this.connectionManager.databaseName]: this.options.includedTables },
+      filename: this.binLogPosition.filename,
+      position: this.binLogPosition.offset,
+      serverId: this.options.serverId
+    } satisfies StartOptions);
+
+    return new Promise<void>((resolve, reject) => {
+      // Handle an edge case where the listener has already been stopped before completing startup
+      if (this.isStopped) {
+        this.logger.info('BinLog listener was stopped before startup completed.');
+        resolve();
+      }
+
+      this.zongji.on('error', (error) => {
+        if (!this.isStopped) {
+          this.logger.error('Binlog listener error:', error);
+          this.stop();
+          reject(error);
+        } else {
+          this.logger.warn('Binlog listener error during shutdown:', error);
+        }
+      });
+
+      this.processingQueue.error((error) => {
+        if (!this.isStopped) {
+          this.logger.error('BinlogEvent processing error:', error);
+          this.stop();
+          reject(error);
+        } else {
+          this.logger.warn('BinlogEvent processing error during shutdown:', error);
+        }
+      });
+
+      this.zongji.on('stopped', () => {
+        resolve();
+        this.logger.info('BinLog listener stopped. Replication ended.');
+      });
+    });
+  }
+
+  public stop(): void {
+    if (!this.isStopped) {
+      this.zongji.stop();
+      this.processingQueue.kill();
+    }
+  }
+
+  private get isStopped(): boolean {
+    return this.zongji.stopped;
+  }
+
+  private createZongjiListener(): ZongJi {
+    const zongji = this.connectionManager.createBinlogListener();
+
+    zongji.on('binlog', async (evt) => {
+      this.logger.info(`Received Binlog event:${evt.getEventName()}`);
+      this.processingQueue.push(evt);
+      this.queueMemoryUsage += evt.size;
+
+      // When the processing queue grows past the threshold, we pause the binlog listener
+      if (this.isQueueOverCapacity()) {
+        this.logger.info(
+          `Binlog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing Binlog listener.`
+        );
+        zongji.pause();
+        const resumeTimeoutPromise = new Promise((resolve) => {
+          setTimeout(() => resolve('timeout'), MAX_QUEUE_PAUSE_TIME_MS);
+        });
+
+        await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]);
+
+        this.logger.info(`Binlog processing queue backlog cleared. Resuming Binlog listener.`);
+        zongji.resume();
+      }
+    });
+
+    zongji.on('ready', async () => {
+      // Set a heartbeat interval for the Zongji replication connection
+      // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown
+      // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket.
+      await new Promise((resolve, reject) => {
+        this.zongji.connection.query(
+          // In nanoseconds, 10^9 = 1s
+          'set @master_heartbeat_period=28*1000000000',
+          (error: any, results: any, fields: any) => {
+            if (error) {
+              reject(error);
+            } else {
+              this.logger.info('Successfully set up replication connection heartbeat...');
+              resolve(results);
+            }
+          }
+        );
+      });
+
+      // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat.
+      // The timeout here must be greater than the master_heartbeat_period.
+      const socket = this.zongji.connection._socket!;
+      socket.setTimeout(60_000, () => {
+        this.logger.info('Destroying socket due to replication connection timeout.');
+        socket.destroy(new Error('Replication connection timeout.'));
+      });
+      this.logger.info(
+        `BinLog listener setup complete. Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`
+      );
+    });
+
+    return zongji;
+  }
+
+  private createQueueWorker() {
+    return async (evt: BinLogEvent) => {
+      switch (true) {
+        case zongji_utils.eventIsGTIDLog(evt):
+          this.currentGTID = common.ReplicatedGTID.fromBinLogEvent({
+            raw_gtid: {
+              server_id: evt.serverId,
+              transaction_range: evt.transactionRange
+            },
+            position: {
+              filename: this.binLogPosition.filename,
+              offset: evt.nextPosition
+            }
+          });
+          await this.eventHandler.onTransactionStart({ timestamp: new Date(evt.timestamp) });
+          break;
+        case zongji_utils.eventIsRotation(evt):
+          this.binLogPosition.filename = evt.binlogName;
+          this.binLogPosition.offset = evt.position;
+          await this.eventHandler.onRotate();
+          break;
+        case zongji_utils.eventIsWriteMutation(evt):
+          await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]);
+          break;
+        case zongji_utils.eventIsUpdateMutation(evt):
+          await this.eventHandler.onUpdate(
+            evt.rows.map((row) => row.after),
+            evt.rows.map((row) => row.before),
+            evt.tableMap[evt.tableId]
+          );
+          break;
+        case zongji_utils.eventIsDeleteMutation(evt):
+          await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]);
+          break;
+        case zongji_utils.eventIsXid(evt):
+          const LSN = new common.ReplicatedGTID({
+            raw_gtid: this.currentGTID!.raw,
+            position: {
+              filename: this.binLogPosition.filename,
+              offset: evt.nextPosition
+            }
+          }).comparable;
+          await this.eventHandler.onCommit(LSN);
+          break;
+      }
+
+      this.queueMemoryUsage -= evt.size;
+    };
+  }
+
+  isQueueOverCapacity(): boolean {
+    return this.queueMemoryUsage >= this.queueMemoryLimit;
+  }
+}

package/src/replication/zongji/zongji-utils.ts

@@ -1,9 +1,10 @@
 import {
   BinLogEvent,
   BinLogGTIDLogEvent,
-  BinLogMutationEvent,
+  BinLogRowEvent,
   BinLogRotationEvent,
-  BinLogUpdateEvent,
+  BinLogTableMapEvent,
+  BinLogRowUpdateEvent,
   BinLogXidEvent
 } from '@powersync/mysql-zongji';
 
@@ -11,6 +12,10 @@ export function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent
   return event.getEventName() == 'gtidlog';
 }
 
+export function eventIsTableMap(event: BinLogEvent): event is BinLogTableMapEvent {
+  return event.getEventName() == 'tablemap';
+}
+
 export function eventIsXid(event: BinLogEvent): event is BinLogXidEvent {
   return event.getEventName() == 'xid';
 }
@@ -19,14 +24,14 @@ export function eventIsRotation(event: BinLogEvent): event is BinLogRotationEven
   return event.getEventName() == 'rotate';
 }
 
-export function eventIsWriteMutation(event: BinLogEvent): event is BinLogMutationEvent {
+export function eventIsWriteMutation(event: BinLogEvent): event is BinLogRowEvent {
   return event.getEventName() == 'writerows';
 }
 
-export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogMutationEvent {
+export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogRowEvent {
   return event.getEventName() == 'deleterows';
 }
 
-export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogUpdateEvent {
+export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogRowUpdateEvent {
   return event.getEventName() == 'updaterows';
 }

package/src/types/types.ts

@@ -23,6 +23,8 @@ export interface NormalizedMySQLConnectionConfig {
   client_private_key?: string;
 
   lookup?: LookupFunction;
+
+  binlog_queue_memory_limit: number;
 }
 
 export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.and(
@@ -40,7 +42,9 @@ export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.a
     client_certificate: t.string.optional(),
     client_private_key: t.string.optional(),
 
-    reject_ip_ranges: t.array(t.string).optional()
+    reject_ip_ranges: t.array(t.string).optional(),
+    // The combined size of binlog events that can be queued in memory before throttling is applied.
+    binlog_queue_memory_limit: t.number.optional()
   })
 );
 
@@ -114,6 +118,9 @@ export function normalizeConnectionConfig(options: MySQLConnectionConfig): Norma
 
     server_id: options.server_id ?? 1,
 
+    // Binlog processing queue memory limit before throttling is applied.
+    binlog_queue_memory_limit: options.binlog_queue_memory_limit ?? 50,
+
     lookup
   };
 }

package/src/utils/mysql-utils.ts

@@ -41,6 +41,7 @@ export function createPool(config: types.NormalizedMySQLConnectionConfig, option
   return mysql.createPool({
     host: config.hostname,
     user: config.username,
+    port: config.port,
     password: config.password,
     database: config.database,
     ssl: hasSSLOptions ? sslOptions : undefined,

package/test/src/BinLogListener.test.ts

@@ -0,0 +1,161 @@
+import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest';
+import { BinLogEventHandler, BinLogListener, Row } from '@module/replication/zongji/BinLogListener.js';
+import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
+import { v4 as uuid } from 'uuid';
+import * as common from '@module/common/common-index.js';
+import { createRandomServerId } from '@module/utils/mysql-utils.js';
+import { TableMapEntry } from '@powersync/mysql-zongji';
+import crypto from 'crypto';
+
+describe('BinlogListener tests', () => {
+  const MAX_QUEUE_CAPACITY_MB = 1;
+  const BINLOG_LISTENER_CONNECTION_OPTIONS = {
+    ...TEST_CONNECTION_OPTIONS,
+    binlog_queue_memory_limit: MAX_QUEUE_CAPACITY_MB
+  };
+
+  let connectionManager: MySQLConnectionManager;
+  let eventHandler: TestBinLogEventHandler;
+  let binLogListener: BinLogListener;
+
+  beforeEach(async () => {
+    connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {});
+    const connection = await connectionManager.getConnection();
+    await clearTestDb(connection);
+    await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
+    connection.release();
+    const fromGTID = await getFromGTID(connectionManager);
+
+    eventHandler = new TestBinLogEventHandler();
+    binLogListener = new BinLogListener({
+      connectionManager: connectionManager,
+      eventHandler: eventHandler,
+      startPosition: fromGTID.position,
+      includedTables: ['test_DATA'],
+      serverId: createRandomServerId(1)
+    });
+  });
+
+  afterEach(async () => {
+    await connectionManager.end();
+  });
+
+  test('Stop binlog listener', async () => {
+    const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
+    const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill');
+
+    const startPromise = binLogListener.start();
+    setTimeout(async () => binLogListener.stop(), 50);
+
+    await expect(startPromise).resolves.toBeUndefined();
+    expect(stopSpy).toHaveBeenCalled();
+    expect(queueStopSpy).toHaveBeenCalled();
+  });
+
+  test('Pause Zongji binlog listener when processing queue reaches maximum memory size', async () => {
+    const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause');
+    const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume');
+
+    // Pause the event handler to force a backlog on the processing queue
+    eventHandler.pause();
+
+    const ROW_COUNT = 10;
+    await insertRows(connectionManager, ROW_COUNT);
+
+    const startPromise = binLogListener.start();
+
+    // Wait for listener to pause due to queue reaching capacity
+    await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 });
+
+    expect(binLogListener.isQueueOverCapacity()).toBeTruthy();
+    // Resume event processing
+    eventHandler.unpause!();
+
+    await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 });
+    binLogListener.stop();
+    await expect(startPromise).resolves.toBeUndefined();
+    // Confirm resume was called after unpausing
+    expect(resumeSpy).toHaveBeenCalled();
+  });
+
+  test('Binlog events are correctly forwarded to provided binlog events handler', async () => {
+    const startPromise = binLogListener.start();
+
+    const ROW_COUNT = 10;
+    await insertRows(connectionManager, ROW_COUNT);
+    await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 });
+    expect(eventHandler.commitCount).equals(ROW_COUNT);
+
+    await updateRows(connectionManager);
+    await vi.waitFor(() => expect(eventHandler.rowsUpdated).equals(ROW_COUNT), { timeout: 5000 });
+
+    await deleteRows(connectionManager);
+    await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 });
+
+    binLogListener.stop();
+    await expect(startPromise).resolves.toBeUndefined();
+  });
+});
+
+async function getFromGTID(connectionManager: MySQLConnectionManager) {
+  const connection = await connectionManager.getConnection();
+  const fromGTID = await common.readExecutedGtid(connection);
+  connection.release();
+
+  return fromGTID;
+}
+
+async function insertRows(connectionManager: MySQLConnectionManager, count: number) {
+  for (let i = 0; i < count; i++) {
+    await connectionManager.query(
+      `INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i} ${crypto.randomBytes(100_000).toString('hex')}')`
+    );
+  }
+}
+
+async function updateRows(connectionManager: MySQLConnectionManager) {
+  await connectionManager.query(`UPDATE test_DATA SET description='updated'`);
+}
+
+async function deleteRows(connectionManager: MySQLConnectionManager) {
+  await connectionManager.query(`DELETE FROM test_DATA`);
+}
+
+class TestBinLogEventHandler implements BinLogEventHandler {
+  rowsWritten = 0;
+  rowsUpdated = 0;
+  rowsDeleted = 0;
+  commitCount = 0;
+
+  unpause: ((value: void | PromiseLike<void>) => void) | undefined;
+  private pausedPromise: Promise<void> | undefined;
+
+  pause() {
+    this.pausedPromise = new Promise((resolve) => {
+      this.unpause = resolve;
+    });
+  }
+
+  async onWrite(rows: Row[], tableMap: TableMapEntry) {
+    if (this.pausedPromise) {
+      await this.pausedPromise;
+    }
+    this.rowsWritten = this.rowsWritten + rows.length;
+  }
+
+  async onUpdate(afterRows: Row[], beforeRows: Row[], tableMap: TableMapEntry) {
+    this.rowsUpdated = this.rowsUpdated + afterRows.length;
+  }
+
+  async onDelete(rows: Row[], tableMap: TableMapEntry) {
+    this.rowsDeleted = this.rowsDeleted + rows.length;
+  }
+
+  async onCommit(lsn: string) {
+    this.commitCount++;
+  }
+
+  async onTransactionStart(options: { timestamp: Date }) {}
+  async onRotate() {}
+}

package/test/src/BinLogStream.test.ts

@@ -1,11 +1,10 @@
 import { storage } from '@powersync/service-core';
 import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
+import { ReplicationMetric } from '@powersync/service-types';
 import { v4 as uuid } from 'uuid';
 import { describe, expect, test } from 'vitest';
 import { BinlogStreamTestContext } from './BinlogStreamUtils.js';
-import { env } from './env.js';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
-import { ReplicationMetric } from '@powersync/service-types';
+import { describeWithStorage } from './util.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -14,12 +13,8 @@ bucket_definitions:
       - SELECT id, description FROM "test_data"
 `;
 
-describe.skipIf(!env.TEST_MONGO_STORAGE)(' Binlog stream - mongodb', { timeout: 20_000 }, function () {
-  defineBinlogStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-});
-
-describe.skipIf(!env.TEST_POSTGRES_STORAGE)(' Binlog stream - postgres', { timeout: 20_000 }, function () {
-  defineBinlogStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+describe('BigLog stream', () => {
+  describeWithStorage({ timeout: 20_000 }, defineBinlogStreamTests);
 });
 
 function defineBinlogStreamTests(factory: storage.TestStorageFactory) {

package/test/src/mysql-to-sqlite.test.ts

@@ -3,7 +3,7 @@ import { afterAll, describe, expect, test } from 'vitest';
 import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js';
 import * as common from '@module/common/common-index.js';
-import ZongJi, { BinLogEvent } from '@powersync/mysql-zongji';
+import { BinLogEvent, ZongJi } from '@powersync/mysql-zongji';
 import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
 import { toColumnDescriptors } from '@module/common/common-index.js';
 

package/test/src/util.ts

@@ -4,6 +4,8 @@ import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import mysqlPromise from 'mysql2/promise';
 import { env } from './env.js';
+import { describe, TestOptions } from 'vitest';
+import { TestStorageFactory } from '@powersync/service-core';
 
 export const TEST_URI = env.MYSQL_TEST_URI;
 
@@ -21,6 +23,16 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTes
   url: env.PG_STORAGE_TEST_URL
 });
 
+export function describeWithStorage(options: TestOptions, fn: (factory: TestStorageFactory) => void) {
+  describe.skipIf(!env.TEST_MONGO_STORAGE)(`mongodb storage`, options, function () {
+    fn(INITIALIZED_MONGO_STORAGE_FACTORY);
+  });
+
+  describe.skipIf(!env.TEST_POSTGRES_STORAGE)(`postgres storage`, options, function () {
+    fn(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+  });
+}
+
 export async function clearTestDb(connection: mysqlPromise.Connection) {
   const version = await getMySQLVersion(connection);
   if (isVersionAtLeast(version, '8.4.0')) {

package/test/tsconfig.json

@@ -13,7 +13,7 @@
       "@core-tests/*": ["../../../packages/service-core/test/src/*"]
     }
   },
-  "include": ["src", "../src/replication/zongji/zongji.d.ts"],
+  "include": ["src"],
   "references": [
     {
       "path": "../"