@powersync/service-module-mysql 0.0.0-dev-20241015210820

package/src/replication/BinLogStream.ts

@@ -0,0 +1,547 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as sync_rules from '@powersync/service-sync-rules';
+import async from 'async';
+
+import { framework, getUuidReplicaIdentityBson, Metrics, storage } from '@powersync/service-core';
+import mysql from 'mysql2';
+
+import { BinLogEvent } from '@powersync/mysql-zongji';
+import * as common from '../common/common-index.js';
+import * as zongji_utils from './zongji/zongji-utils.js';
+import { MySQLConnectionManager } from './MySQLConnectionManager.js';
+import { ReplicatedGTID } from '../common/common-index.js';
+import mysqlPromise from 'mysql2/promise';
+
+export interface BinLogStreamOptions {
+  connections: MySQLConnectionManager;
+  storage: storage.SyncRulesBucketStorage;
+  abortSignal: AbortSignal;
+}
+
+interface MysqlRelId {
+  schema: string;
+  name: string;
+}
+
+interface WriteChangePayload {
+  type: storage.SaveOperationTag;
+  data: Data;
+  previous_data?: Data;
+  database: string;
+  table: string;
+  sourceTable: storage.SourceTable;
+}
+
+export type Data = Record<string, any>;
+
+/**
+ * MySQL does not have same relation structure. Just returning unique key as string.
+ * @param source
+ */
+function getMysqlRelId(source: MysqlRelId): string {
+  return `${source.schema}.${source.name}`;
+}
+
+export class BinLogStream {
+  private readonly syncRules: sync_rules.SqlSyncRules;
+  private readonly groupId: number;
+
+  private readonly storage: storage.SyncRulesBucketStorage;
+
+  private readonly connections: MySQLConnectionManager;
+
+  private abortSignal: AbortSignal;
+
+  private tableCache = new Map<string | number, storage.SourceTable>();
+
+  constructor(protected options: BinLogStreamOptions) {
+    this.storage = options.storage;
+    this.connections = options.connections;
+    this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: this.defaultSchema });
+    this.groupId = options.storage.group_id;
+    this.abortSignal = options.abortSignal;
+  }
+
+  get connectionTag() {
+    return this.connections.connectionTag;
+  }
+
+  get connectionId() {
+    // Default to 1 if not set
+    return this.connections.connectionId ? Number.parseInt(this.connections.connectionId) : 1;
+  }
+
+  get stopped() {
+    return this.abortSignal.aborted;
+  }
+
+  get defaultSchema() {
+    return this.connections.databaseName;
+  }
+
+  async handleRelation(batch: storage.BucketStorageBatch, entity: storage.SourceEntityDescriptor, snapshot: boolean) {
+    const result = await this.storage.resolveTable({
+      group_id: this.groupId,
+      connection_id: this.connectionId,
+      connection_tag: this.connectionTag,
+      entity_descriptor: entity,
+      sync_rules: this.syncRules
+    });
+    this.tableCache.set(entity.objectId, result.table);
+
+    // Drop conflicting tables. This includes for example renamed tables.
+    await batch.drop(result.dropTables);
+
+    // Snapshot if:
+    // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
+    // 2. Snapshot is not already done, AND:
+    // 3. The table is used in sync rules.
+    const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
+
+    if (shouldSnapshot) {
+      // Truncate this table, in case a previous snapshot was interrupted.
+      await batch.truncate([result.table]);
+
+      let gtid: common.ReplicatedGTID;
+      // Start the snapshot inside a transaction.
+      // We use a dedicated connection for this.
+      const connection = await this.connections.getStreamingConnection();
+      const promiseConnection = (connection as mysql.Connection).promise();
+      try {
+        await promiseConnection.query('BEGIN');
+        try {
+          gtid = await common.readExecutedGtid(promiseConnection);
+          await this.snapshotTable(connection.connection, batch, result.table);
+          await promiseConnection.query('COMMIT');
+        } catch (e) {
+          await promiseConnection.query('ROLLBACK');
+          throw e;
+        }
+      } finally {
+        connection.release();
+      }
+      const [table] = await batch.markSnapshotDone([result.table], gtid.comparable);
+      return table;
+    }
+
+    return result.table;
+  }
+
+  async getQualifiedTableNames(
+    batch: storage.BucketStorageBatch,
+    tablePattern: sync_rules.TablePattern
+  ): Promise<storage.SourceTable[]> {
+    if (tablePattern.connectionTag != this.connectionTag) {
+      return [];
+    }
+
+    let tableRows: any[];
+    const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined;
+    if (tablePattern.isWildcard) {
+      const result = await this.connections.query(
+        `SELECT TABLE_NAME
+FROM information_schema.tables
+WHERE TABLE_SCHEMA = ? AND TABLE_NAME LIKE ?;
+`,
+        [tablePattern.schema, tablePattern.tablePattern]
+      );
+      tableRows = result[0];
+    } else {
+      const result = await this.connections.query(
+        `SELECT TABLE_NAME
+FROM information_schema.tables
+WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?;
+`,
+        [tablePattern.schema, tablePattern.tablePattern]
+      );
+      tableRows = result[0];
+    }
+    let tables: storage.SourceTable[] = [];
+
+    for (let row of tableRows) {
+      const name = row['TABLE_NAME'] as string;
+      if (prefix && !name.startsWith(prefix)) {
+        continue;
+      }
+
+      const result = await this.connections.query(
+        `SELECT 1
+FROM information_schema.tables
+WHERE table_schema = ? AND table_name = ?
+AND table_type = 'BASE TABLE';`,
+        [tablePattern.schema, tablePattern.name]
+      );
+      if (result[0].length == 0) {
+        logger.info(`Skipping ${tablePattern.schema}.${name} - no table exists/is not a base table`);
+        continue;
+      }
+
+      const connection = await this.connections.getConnection();
+      const replicationColumns = await common.getReplicationIdentityColumns({
+        connection: connection,
+        schema: tablePattern.schema,
+        table_name: tablePattern.name
+      });
+      connection.release();
+
+      const table = await this.handleRelation(
+        batch,
+        {
+          name,
+          schema: tablePattern.schema,
+          objectId: getMysqlRelId(tablePattern),
+          replicationColumns: replicationColumns.columns
+        },
+        false
+      );
+
+      tables.push(table);
+    }
+    return tables;
+  }
+
+  /**
+   * Checks if the initial sync has been completed yet.
+   */
+  protected async checkInitialReplicated(): Promise<boolean> {
+    const status = await this.storage.getStatus();
+    if (status.snapshot_done && status.checkpoint_lsn) {
+      logger.info(`Initial replication already done. MySQL appears healthy`);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Does the initial replication of the database tables.
+   *
+   * If (partial) replication was done before on this slot, this clears the state
+   * and starts again from scratch.
+   */
+  async startInitialReplication() {
+    await this.storage.clear();
+    // Replication will be performed in a single transaction on this connection
+    const connection = await this.connections.getStreamingConnection();
+    const promiseConnection = (connection as mysql.Connection).promise();
+    const headGTID = await common.readExecutedGtid(promiseConnection);
+    logger.info(`Using snapshot checkpoint GTID:: '${headGTID}'`);
+    try {
+      logger.info(`Starting initial replication`);
+      await promiseConnection.query<mysqlPromise.RowDataPacket[]>(
+        'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY'
+      );
+      await promiseConnection.query<mysqlPromise.RowDataPacket[]>('START TRANSACTION');
+      const sourceTables = this.syncRules.getSourceTables();
+      await this.storage.startBatch(
+        { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema },
+        async (batch) => {
+          for (let tablePattern of sourceTables) {
+            const tables = await this.getQualifiedTableNames(batch, tablePattern);
+            for (let table of tables) {
+              await this.snapshotTable(connection as mysql.Connection, batch, table);
+              await batch.markSnapshotDone([table], headGTID.comparable);
+              await framework.container.probes.touch();
+            }
+          }
+          await batch.commit(headGTID.comparable);
+        }
+      );
+      logger.info(`Initial replication done`);
+      await promiseConnection.query('COMMIT');
+    } catch (e) {
+      await promiseConnection.query('ROLLBACK');
+      throw e;
+    } finally {
+      connection.release();
+    }
+  }
+
+  private async snapshotTable(
+    connection: mysql.Connection,
+    batch: storage.BucketStorageBatch,
+    table: storage.SourceTable
+  ) {
+    logger.info(`Replicating ${table.qualifiedName}`);
+    // TODO count rows and log progress at certain batch sizes
+
+    return new Promise<void>((resolve, reject) => {
+      // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
+      connection
+        .query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${table.schema}.${table.table}`)
+        .stream()
+        .on('error', (err) => {
+          reject(err);
+        })
+        .on('data', async (row) => {
+          connection.pause();
+          const record = common.toSQLiteRow(row);
+
+          await batch.save({
+            tag: storage.SaveOperationTag.INSERT,
+            sourceTable: table,
+            before: undefined,
+            beforeReplicaId: undefined,
+            after: record,
+            afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns)
+          });
+          connection.resume();
+          // TODO: These metrics can probably be reported in batches
+          Metrics.getInstance().rows_replicated_total.add(1);
+        })
+        .on('end', async function () {
+          await batch.flush();
+          resolve();
+        });
+    });
+  }
+
+  async replicate() {
+    try {
+      // If anything errors here, the entire replication process is halted, and
+      // all connections automatically closed, including this one.
+      await this.initReplication();
+      await this.streamChanges();
+    } catch (e) {
+      await this.storage.reportError(e);
+      throw e;
+    }
+  }
+
+  async initReplication() {
+    const connection = await this.connections.getConnection();
+    await common.checkSourceConfiguration(connection);
+    connection.release();
+
+    const initialReplicationCompleted = await this.checkInitialReplicated();
+    if (!initialReplicationCompleted) {
+      await this.startInitialReplication();
+    }
+  }
+
+  private getTable(tableId: string): storage.SourceTable {
+    const table = this.tableCache.get(tableId);
+    if (table == null) {
+      // We should always receive a replication message before the relation is used.
+      // If we can't find it, it's a bug.
+      throw new Error(`Missing relation cache for ${tableId}`);
+    }
+    return table;
+  }
+
+  async streamChanges() {
+    // Auto-activate as soon as initial replication is done
+    await this.storage.autoActivate();
+
+    const connection = await this.connections.getConnection();
+    const { checkpoint_lsn } = await this.storage.getStatus();
+    logger.info(`Last known LSN from storage: ${checkpoint_lsn}`);
+
+    const fromGTID = checkpoint_lsn
+      ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn)
+      : await common.readExecutedGtid(connection);
+    const binLogPositionState = fromGTID.position;
+    connection.release();
+
+    await this.storage.startBatch(
+      { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema },
+      async (batch) => {
+        const zongji = this.connections.createBinlogListener();
+
+        let currentGTID: common.ReplicatedGTID | null = null;
+
+        const queue = async.queue(async (evt: BinLogEvent) => {
+          // State machine
+          switch (true) {
+            case zongji_utils.eventIsGTIDLog(evt):
+              currentGTID = common.ReplicatedGTID.fromBinLogEvent({
+                raw_gtid: {
+                  server_id: evt.serverId,
+                  transaction_range: evt.transactionRange
+                },
+                position: {
+                  filename: binLogPositionState.filename,
+                  offset: evt.nextPosition
+                }
+              });
+              break;
+            case zongji_utils.eventIsRotation(evt):
+              // Update the position
+              binLogPositionState.filename = evt.binlogName;
+              binLogPositionState.offset = evt.position;
+              break;
+            case zongji_utils.eventIsWriteMutation(evt):
+              // TODO, can multiple tables be present?
+              const writeTableInfo = evt.tableMap[evt.tableId];
+              await this.writeChanges(batch, {
+                type: storage.SaveOperationTag.INSERT,
+                data: evt.rows,
+                database: writeTableInfo.parentSchema,
+                table: writeTableInfo.tableName,
+                sourceTable: this.getTable(
+                  getMysqlRelId({
+                    schema: writeTableInfo.parentSchema,
+                    name: writeTableInfo.tableName
+                  })
+                )
+              });
+              break;
+            case zongji_utils.eventIsUpdateMutation(evt):
+              const updateTableInfo = evt.tableMap[evt.tableId];
+              await this.writeChanges(batch, {
+                type: storage.SaveOperationTag.UPDATE,
+                data: evt.rows.map((row) => row.after),
+                previous_data: evt.rows.map((row) => row.before),
+                database: updateTableInfo.parentSchema,
+                table: updateTableInfo.tableName,
+                sourceTable: this.getTable(
+                  getMysqlRelId({
+                    schema: updateTableInfo.parentSchema,
+                    name: updateTableInfo.tableName
+                  })
+                )
+              });
+              break;
+            case zongji_utils.eventIsDeleteMutation(evt):
+              // TODO, can multiple tables be present?
+              const deleteTableInfo = evt.tableMap[evt.tableId];
+              await this.writeChanges(batch, {
+                type: storage.SaveOperationTag.DELETE,
+                data: evt.rows,
+                database: deleteTableInfo.parentSchema,
+                table: deleteTableInfo.tableName,
+                // TODO cleanup
+                sourceTable: this.getTable(
+                  getMysqlRelId({
+                    schema: deleteTableInfo.parentSchema,
+                    name: deleteTableInfo.tableName
+                  })
+                )
+              });
+              break;
+            case zongji_utils.eventIsXid(evt):
+              Metrics.getInstance().transactions_replicated_total.add(1);
+              // Need to commit with a replicated GTID with updated next position
+              await batch.commit(
+                new common.ReplicatedGTID({
+                  raw_gtid: currentGTID!.raw,
+                  position: {
+                    filename: binLogPositionState.filename,
+                    offset: evt.nextPosition
+                  }
+                }).comparable
+              );
+              currentGTID = null;
+              // chunks_replicated_total.add(1);
+              break;
+          }
+        }, 1);
+
+        zongji.on('binlog', (evt: BinLogEvent) => {
+          logger.info(`Pushing Binlog event ${evt.getEventName()}`);
+          queue.push(evt);
+        });
+
+        logger.info(`Starting replication from ${binLogPositionState.filename}:${binLogPositionState.offset}`);
+        zongji.start({
+          includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
+          excludeEvents: [],
+          filename: binLogPositionState.filename,
+          position: binLogPositionState.offset
+        });
+
+        // Forever young
+        await new Promise<void>((resolve, reject) => {
+          queue.error((error) => {
+            zongji.stop();
+            queue.kill();
+            reject(error);
+          });
+          this.abortSignal.addEventListener(
+            'abort',
+            async () => {
+              zongji.stop();
+              queue.kill();
+              if (!queue.length) {
+                await queue.drain();
+              }
+              resolve();
+            },
+            { once: true }
+          );
+        });
+      }
+    );
+  }
+
+  private async writeChanges(
+    batch: storage.BucketStorageBatch,
+    msg: {
+      type: storage.SaveOperationTag;
+      data: Data[];
+      previous_data?: Data[];
+      database: string;
+      table: string;
+      sourceTable: storage.SourceTable;
+    }
+  ): Promise<storage.FlushedResult | null> {
+    for (const [index, row] of msg.data.entries()) {
+      await this.writeChange(batch, {
+        ...msg,
+        data: row,
+        previous_data: msg.previous_data?.[index]
+      });
+    }
+    return null;
+  }
+
+  private async writeChange(
+    batch: storage.BucketStorageBatch,
+    payload: WriteChangePayload
+  ): Promise<storage.FlushedResult | null> {
+    switch (payload.type) {
+      case storage.SaveOperationTag.INSERT:
+        Metrics.getInstance().rows_replicated_total.add(1);
+        const record = common.toSQLiteRow(payload.data);
+        return await batch.save({
+          tag: storage.SaveOperationTag.INSERT,
+          sourceTable: payload.sourceTable,
+          before: undefined,
+          beforeReplicaId: undefined,
+          after: record,
+          afterReplicaId: getUuidReplicaIdentityBson(record, payload.sourceTable.replicaIdColumns)
+        });
+      case storage.SaveOperationTag.UPDATE:
+        Metrics.getInstance().rows_replicated_total.add(1);
+        // "before" may be null if the replica id columns are unchanged
+        // It's fine to treat that the same as an insert.
+        const beforeUpdated = payload.previous_data ? common.toSQLiteRow(payload.previous_data) : undefined;
+        const after = common.toSQLiteRow(payload.data);
+
+        return await batch.save({
+          tag: storage.SaveOperationTag.UPDATE,
+          sourceTable: payload.sourceTable,
+          before: beforeUpdated,
+          beforeReplicaId: beforeUpdated
+            ? getUuidReplicaIdentityBson(beforeUpdated, payload.sourceTable.replicaIdColumns)
+            : undefined,
+          after: common.toSQLiteRow(payload.data),
+          afterReplicaId: getUuidReplicaIdentityBson(after, payload.sourceTable.replicaIdColumns)
+        });
+
+      case storage.SaveOperationTag.DELETE:
+        Metrics.getInstance().rows_replicated_total.add(1);
+        const beforeDeleted = common.toSQLiteRow(payload.data);
+
+        return await batch.save({
+          tag: storage.SaveOperationTag.DELETE,
+          sourceTable: payload.sourceTable,
+          before: beforeDeleted,
+          beforeReplicaId: getUuidReplicaIdentityBson(beforeDeleted, payload.sourceTable.replicaIdColumns),
+          after: undefined,
+          afterReplicaId: undefined
+        });
+      default:
+        return null;
+    }
+  }
+}

package/src/replication/MySQLConnectionManager.ts

@@ -0,0 +1,104 @@
+import { NormalizedMySQLConnectionConfig } from '../types/types.js';
+import mysqlPromise from 'mysql2/promise';
+import mysql, { RowDataPacket } from 'mysql2';
+import * as mysql_utils from '../utils/mysql_utils.js';
+import ZongJi from '@powersync/mysql-zongji';
+
+export class MySQLConnectionManager {
+  /**
+   *  Pool that can create streamable connections
+   */
+  private readonly pool: mysql.Pool;
+  /**
+   *  Pool that can create promise-based connections
+   */
+  private readonly promisePool: mysqlPromise.Pool;
+
+  private binlogListeners: ZongJi[] = [];
+
+  constructor(
+    public options: NormalizedMySQLConnectionConfig,
+    public poolOptions: mysqlPromise.PoolOptions
+  ) {
+    // The pool is lazy - no connections are opened until a query is performed.
+    this.pool = mysql_utils.createPool(options, poolOptions);
+    this.promisePool = this.pool.promise();
+  }
+
+  public get connectionTag() {
+    return this.options.tag;
+  }
+
+  public get connectionId() {
+    return this.options.id;
+  }
+
+  public get databaseName() {
+    return this.options.database;
+  }
+
+  /**
+   * Create a new replication listener
+   */
+  createBinlogListener(): ZongJi {
+    const listener = new ZongJi({
+      host: this.options.hostname,
+      user: this.options.username,
+      password: this.options.password
+    });
+
+    this.binlogListeners.push(listener);
+
+    return listener;
+  }
+
+  /**
+   *  Run a query using a connection from the pool
+   *  A promise with the result is returned
+   *  @param query
+   *  @param params
+   */
+  async query(query: string, params?: any[]) {
+    return this.promisePool.query<RowDataPacket[]>(query, params);
+  }
+
+  /**
+   *  Get a streamable connection from this manager's pool
+   *  The connection should be released when it is no longer needed
+   */
+  async getStreamingConnection(): Promise<mysql.PoolConnection> {
+    return new Promise((resolve, reject) => {
+      this.pool.getConnection((err, connection) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(connection);
+        }
+      });
+    });
+  }
+
+  /**
+   *  Get a promise connection from this manager's pool
+   *  The connection should be released when it is no longer needed
+   */
+  async getConnection(): Promise<mysqlPromise.PoolConnection> {
+    return this.promisePool.getConnection();
+  }
+
+  async end(): Promise<void> {
+    for (const listener of this.binlogListeners) {
+      listener.stop();
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      this.pool.end((err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+}

package/src/replication/MySQLConnectionManagerFactory.ts

@@ -0,0 +1,28 @@
+import { logger } from '@powersync/lib-services-framework';
+import mysql from 'mysql2/promise';
+import { MySQLConnectionManager } from './MySQLConnectionManager.js';
+import { ResolvedConnectionConfig } from '../types/types.js';
+
+export class MySQLConnectionManagerFactory {
+  private readonly connectionManagers: MySQLConnectionManager[];
+  private readonly connectionConfig: ResolvedConnectionConfig;
+
+  constructor(connectionConfig: ResolvedConnectionConfig) {
+    this.connectionConfig = connectionConfig;
+    this.connectionManagers = [];
+  }
+
+  create(poolOptions: mysql.PoolOptions) {
+    const manager = new MySQLConnectionManager(this.connectionConfig, poolOptions);
+    this.connectionManagers.push(manager);
+    return manager;
+  }
+
+  async shutdown() {
+    logger.info('Shutting down MySQL connection Managers...');
+    for (const manager of this.connectionManagers) {
+      await manager.end();
+    }
+    logger.info('MySQL connection Managers shutdown completed.');
+  }
+}