@powersync/service-module-mysql 0.7.4 → 0.9.0

package/src/replication/BinLogStream.ts

@@ -10,7 +10,9 @@ import {
   ColumnDescriptor,
   framework,
   getUuidReplicaIdentityBson,
+  InternalOpId,
   MetricsEngine,
+  SourceTable,
   storage
 } from '@powersync/service-core';
 import mysql from 'mysql2';
@@ -18,10 +20,10 @@ import mysqlPromise from 'mysql2/promise';
 
 import { TableMapEntry } from '@powersync/mysql-zongji';
 import * as common from '../common/common-index.js';
-import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js';
+import { createRandomServerId, qualifiedMySQLTable } from '../utils/mysql-utils.js';
 import { MySQLConnectionManager } from './MySQLConnectionManager.js';
 import { ReplicationMetric } from '@powersync/service-types';
-import { BinLogEventHandler, BinLogListener, Row } from './zongji/BinLogListener.js';
+import { BinLogEventHandler, BinLogListener, Row, SchemaChange, SchemaChangeType } from './zongji/BinLogListener.js';
 
 export interface BinLogStreamOptions {
   connections: MySQLConnectionManager;
@@ -31,11 +33,6 @@ export interface BinLogStreamOptions {
   logger?: Logger;
 }
 
-interface MysqlRelId {
-  schema: string;
-  name: string;
-}
-
 interface WriteChangePayload {
   type: storage.SaveOperationTag;
   row: Row;
@@ -53,11 +50,14 @@ export class BinlogConfigurationError extends Error {
 }
 
 /**
- * MySQL does not have same relation structure. Just returning unique key as string.
- * @param source
+ * Unlike Postgres' relation id, MySQL's tableId is only guaranteed to be unique and stay the same
+ * in the context of a single replication session.
+ * Instead, we create a unique key by combining the source schema and table name
+ * @param schema
+ * @param tableName
  */
-function getMysqlRelId(source: MysqlRelId): string {
-  return `${source.schema}.${source.name}`;
+function createTableId(schema: string, tableName: string): string {
+  return `${schema}.${tableName}`;
 }
 
 export class BinLogStream {
@@ -68,11 +68,11 @@ export class BinLogStream {
 
   private readonly connections: MySQLConnectionManager;
 
-  private abortSignal: AbortSignal;
+  private readonly abortSignal: AbortSignal;
 
-  private tableCache = new Map<string | number, storage.SourceTable>();
+  private readonly logger: Logger;
 
-  private logger: Logger;
+  private tableCache = new Map<string | number, storage.SourceTable>();
 
   /**
    * Time of the oldest uncommitted change, according to the source db.
@@ -83,7 +83,7 @@ export class BinLogStream {
    * Keep track of whether we have done a commit or keepalive yet.
    * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
    */
-  private isStartingReplication = true;
+  isStartingReplication = true;
 
   constructor(private options: BinLogStreamOptions) {
     this.logger = options.logger ?? defaultLogger;
@@ -134,15 +134,15 @@ export class BinLogStream {
       entity_descriptor: entity,
       sync_rules: this.syncRules
     });
-    // objectId is always defined for mysql
+    // Since we create the objectId ourselves, this is always defined
     this.tableCache.set(entity.objectId!, result.table);
 
-    // Drop conflicting tables. This includes for example renamed tables.
+    // Drop conflicting tables. In the MySQL case with ObjectIds created from the table name, renames cannot be detected by the storage.
     await batch.drop(result.dropTables);
 
     // Snapshot if:
     // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
-    // 2. Snapshot is not already done, AND:
+    // 2. Snapshot is not done yet, AND:
     // 3. The table is used in sync rules.
     const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
 
@@ -158,10 +158,10 @@ export class BinLogStream {
       const promiseConnection = (connection as mysql.Connection).promise();
       try {
         await promiseConnection.query(`SET time_zone = '+00:00'`);
-        await promiseConnection.query('BEGIN');
+        await promiseConnection.query('START TRANSACTION');
         try {
           gtid = await common.readExecutedGtid(promiseConnection);
-          await this.snapshotTable(connection.connection, batch, result.table);
+          await this.snapshotTable(connection as mysql.Connection, batch, result.table);
           await promiseConnection.query('COMMIT');
         } catch (e) {
           await this.tryRollback(promiseConnection);
@@ -185,62 +185,21 @@ export class BinLogStream {
       return [];
     }
 
-    let tableRows: any[];
-    const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined;
-    if (tablePattern.isWildcard) {
-      const result = await this.connections.query(
-        `SELECT TABLE_NAME
-FROM information_schema.tables
-WHERE TABLE_SCHEMA = ? AND TABLE_NAME LIKE ?;
-`,
-        [tablePattern.schema, tablePattern.tablePattern]
-      );
-      tableRows = result[0];
-    } else {
-      const result = await this.connections.query(
-        `SELECT TABLE_NAME
-FROM information_schema.tables
-WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?;
-`,
-        [tablePattern.schema, tablePattern.tablePattern]
-      );
-      tableRows = result[0];
-    }
-    let tables: storage.SourceTable[] = [];
-
-    for (let row of tableRows) {
-      const name = row['TABLE_NAME'] as string;
-      if (prefix && !name.startsWith(prefix)) {
-        continue;
-      }
-
-      const result = await this.connections.query(
-        `SELECT 1
-FROM information_schema.tables
-WHERE table_schema = ? AND table_name = ?
-AND table_type = 'BASE TABLE';`,
-        [tablePattern.schema, tablePattern.name]
-      );
-      if (result[0].length == 0) {
-        this.logger.info(`Skipping ${tablePattern.schema}.${name} - no table exists/is not a base table`);
-        continue;
-      }
+    const connection = await this.connections.getConnection();
+    const matchedTables: string[] = await common.getTablesFromPattern(connection, tablePattern);
+    connection.release();
 
-      const connection = await this.connections.getConnection();
-      const replicationColumns = await common.getReplicationIdentityColumns({
-        connection: connection,
-        schema: tablePattern.schema,
-        table_name: tablePattern.name
-      });
-      connection.release();
+    let tables: storage.SourceTable[] = [];
+    for (const matchedTable of matchedTables) {
+      const replicaIdColumns = await this.getReplicaIdColumns(matchedTable, tablePattern.schema);
 
       const table = await this.handleRelation(
         batch,
         {
-          name,
+          name: matchedTable,
           schema: tablePattern.schema,
-          objectId: getMysqlRelId(tablePattern),
-          replicationColumns: replicationColumns.columns
+          objectId: createTableId(tablePattern.schema, matchedTable),
+          replicaIdColumns: replicaIdColumns
         },
         false
       );
@@ -251,7 +210,7 @@ AND table_type = 'BASE TABLE';`,
   }
 
   /**
-   * Checks if the initial sync has been completed yet.
+   * Checks if the initial sync has already been completed
    */
   protected async checkInitialReplicated(): Promise<boolean> {
     const status = await this.storage.getStatus();
@@ -260,7 +219,7 @@ AND table_type = 'BASE TABLE';`,
       this.logger.info(`Initial replication already done.`);
 
       if (lastKnowGTID) {
-        // Check if the binlog is still available. If it isn't we need to snapshot again.
+        // Check if the specific binlog file is still available. If it isn't, we need to snapshot again.
         const connection = await this.connections.getConnection();
         try {
           const isAvailable = await common.isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
@@ -294,6 +253,7 @@ AND table_type = 'BASE TABLE';`,
     const promiseConnection = (connection as mysql.Connection).promise();
     const headGTID = await common.readExecutedGtid(promiseConnection);
     this.logger.info(`Using snapshot checkpoint GTID: '${headGTID}'`);
+    let lastOp: InternalOpId | null = null;
     try {
       this.logger.info(`Starting initial replication`);
       await promiseConnection.query<mysqlPromise.RowDataPacket[]>(
@@ -303,7 +263,7 @@ AND table_type = 'BASE TABLE';`,
       await promiseConnection.query(`SET time_zone = '+00:00'`);
 
       const sourceTables = this.syncRules.getSourceTables();
-      await this.storage.startBatch(
+      const flushResults = await this.storage.startBatch(
         {
           logger: this.logger,
           zeroLSN: common.ReplicatedGTID.ZERO.comparable,
@@ -322,6 +282,7 @@ AND table_type = 'BASE TABLE';`,
           await batch.commit(headGTID.comparable);
         }
       );
+      lastOp = flushResults?.flushed_op ?? null;
       this.logger.info(`Initial replication done`);
       await promiseConnection.query('COMMIT');
     } catch (e) {
@@ -330,6 +291,15 @@ AND table_type = 'BASE TABLE';`,
     } finally {
       connection.release();
     }
+
+    if (lastOp != null) {
+      // Populate the cache _after_ initial replication, but _before_ we switch to this sync rules.
+      await this.storage.populatePersistentChecksumCache({
+        // No checkpoint yet, but we do have the opId.
+        maxOpId: lastOp,
+        signal: this.abortSignal
+      });
+    }
   }
 
   private async snapshotTable(
@@ -337,11 +307,11 @@ AND table_type = 'BASE TABLE';`,
     batch: storage.BucketStorageBatch,
     table: storage.SourceTable
   ) {
-    this.logger.info(`Replicating ${table.qualifiedName}`);
+    this.logger.info(`Replicating ${qualifiedMySQLTable(table)}`);
     // TODO count rows and log progress at certain batch sizes
 
     // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
-    const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${escapeMysqlTableName(table)}`);
+    const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${qualifiedMySQLTable(table)}`);
     const stream = query.stream();
 
     let columns: Map<string, ColumnDescriptor> | undefined = undefined;
@@ -430,8 +400,6 @@ AND table_type = 'BASE TABLE';`,
   }
 
   async streamChanges() {
-    // Auto-activate as soon as initial replication is done
-    await this.storage.autoActivate();
     const serverId = createRandomServerId(this.storage.group_id);
 
     const connection = await this.connections.getConnection();
@@ -442,7 +410,6 @@ AND table_type = 'BASE TABLE';`,
     const fromGTID = checkpoint_lsn
       ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn)
       : await common.readExecutedGtid(connection);
-    const binLogPositionState = fromGTID.position;
     connection.release();
 
     if (!this.stopped) {
@@ -450,12 +417,10 @@ AND table_type = 'BASE TABLE';`,
         { zeroLSN: common.ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true },
         async (batch) => {
           const binlogEventHandler = this.createBinlogEventHandler(batch);
-          // Only listen for changes to tables in the sync rules
-          const includedTables = [...this.tableCache.values()].map((table) => table.table);
           const binlogListener = new BinLogListener({
             logger: this.logger,
-            includedTables: includedTables,
-            startPosition: binLogPositionState,
+            sourceTables: this.syncRules.getSourceTables(),
+            startGTID: fromGTID,
             connectionManager: this.connections,
             serverId: serverId,
             eventHandler: binlogEventHandler
@@ -463,15 +428,14 @@ AND table_type = 'BASE TABLE';`,
 
           this.abortSignal.addEventListener(
             'abort',
-            () => {
-              this.logger.info('Abort signal received, stopping replication...');
-              binlogListener.stop();
+            async () => {
+              await binlogListener.stop();
             },
             { once: true }
           );
 
-          // Only returns when the replication is stopped or interrupted by an error
           await binlogListener.start();
+          await binlogListener.replicateUntilStopped();
         }
       );
     }
@@ -502,6 +466,12 @@ AND table_type = 'BASE TABLE';`,
           tableEntry: tableMap
         });
       },
+      onKeepAlive: async (lsn: string) => {
+        const didCommit = await batch.keepalive(lsn);
+        if (didCommit) {
+          this.oldestUncommittedChange = null;
+        }
+      },
       onCommit: async (lsn: string) => {
         this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1);
         const didCommit = await batch.commit(lsn, { oldestUncommittedChange: this.oldestUncommittedChange });
@@ -517,10 +487,82 @@ AND table_type = 'BASE TABLE';`,
       },
       onRotate: async () => {
         this.isStartingReplication = false;
+      },
+      onSchemaChange: async (change: SchemaChange) => {
+        await this.handleSchemaChange(batch, change);
       }
     };
   }
 
+  private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void> {
+    if (change.type === SchemaChangeType.RENAME_TABLE) {
+      const fromTableId = createTableId(change.schema, change.table);
+
+      const fromTable = this.tableCache.get(fromTableId);
+      // Old table needs to be cleaned up
+      if (fromTable) {
+        await batch.drop([fromTable]);
+        this.tableCache.delete(fromTableId);
+      }
+      // The new table matched a table in the sync rules
+      if (change.newTable) {
+        await this.handleCreateOrUpdateTable(batch, change.newTable!, change.schema);
+      }
+    } else {
+      const tableId = createTableId(change.schema, change.table);
+
+      const table = this.getTable(tableId);
+
+      switch (change.type) {
+        case SchemaChangeType.ALTER_TABLE_COLUMN:
+        case SchemaChangeType.REPLICATION_IDENTITY:
+          // For these changes, we need to update the table if the replication identity columns have changed.
+          await this.handleCreateOrUpdateTable(batch, change.table, change.schema);
+          break;
+        case SchemaChangeType.TRUNCATE_TABLE:
+          await batch.truncate([table]);
+          break;
+        case SchemaChangeType.DROP_TABLE:
+          await batch.drop([table]);
+          this.tableCache.delete(tableId);
+          break;
+        default:
+          // No action needed for other schema changes
+          break;
+      }
+    }
+  }
+
+  private async getReplicaIdColumns(tableName: string, schema: string) {
+    const connection = await this.connections.getConnection();
+    const replicaIdColumns = await common.getReplicationIdentityColumns({
+      connection,
+      schema,
+      tableName
+    });
+    connection.release();
+
+    return replicaIdColumns.columns;
+  }
+
+  private async handleCreateOrUpdateTable(
+    batch: storage.BucketStorageBatch,
+    tableName: string,
+    schema: string
+  ): Promise<SourceTable> {
+    const replicaIdColumns = await this.getReplicaIdColumns(tableName, schema);
+    return await this.handleRelation(
+      batch,
+      {
+        name: tableName,
+        schema: schema,
+        objectId: createTableId(schema, tableName),
+        replicaIdColumns: replicaIdColumns
+      },
+      true
+    );
+  }
+
   private async writeChanges(
     batch: storage.BucketStorageBatch,
     msg: {
@@ -531,17 +573,20 @@ AND table_type = 'BASE TABLE';`,
     }
   ): Promise<storage.FlushedResult | null> {
     const columns = common.toColumnDescriptors(msg.tableEntry);
+    const tableId = createTableId(msg.tableEntry.parentSchema, msg.tableEntry.tableName);
+
+    let table = this.tableCache.get(tableId);
+    if (table == null) {
+      // This is an insert for a new table that matches a table in the sync rules
+      // We need to create the table in the storage and cache it.
+      table = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema);
+    }
 
     for (const [index, row] of msg.rows.entries()) {
       await this.writeChange(batch, {
         type: msg.type,
         database: msg.tableEntry.parentSchema,
-        sourceTable: this.getTable(
-          getMysqlRelId({
-            schema: msg.tableEntry.parentSchema,
-            name: msg.tableEntry.tableName
-          })
-        ),
+        sourceTable: table!,
         table: msg.tableEntry.tableName,
         columns: columns,
         row: row,
@@ -569,7 +614,7 @@ AND table_type = 'BASE TABLE';`,
         });
       case storage.SaveOperationTag.UPDATE:
         this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
-        // "before" may be null if the replica id columns are unchanged
+        // The previous row may be null if the replica id columns are unchanged.
         // It's fine to treat that the same as an insert.
         const beforeUpdated = payload.previous_row
           ? common.toSQLiteRow(payload.previous_row, payload.columns)
@@ -610,7 +655,7 @@ AND table_type = 'BASE TABLE';`,
         // We don't have anything to compute replication lag with yet.
         return undefined;
       } else {
-        // We don't have any uncommitted changes, so replication is up-to-date.
+        // We don't have any uncommitted changes, so replication is up to date.
         return 0;
       }
     }