@powersync/service-module-mysql 0.1.6 → 0.1.8

package/src/replication/BinLogStream.ts

@@ -11,7 +11,7 @@ import * as zongji_utils from './zongji/zongji-utils.js';
 import { MySQLConnectionManager } from './MySQLConnectionManager.js';
 import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js';
 import mysqlPromise from 'mysql2/promise';
-import { createRandomServerId } from '../utils/mysql-utils.js';
+import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js';
 
 export interface BinLogStreamOptions {
   connections: MySQLConnectionManager;
@@ -114,8 +114,10 @@ export class BinLogStream {
       // Start the snapshot inside a transaction.
       // We use a dedicated connection for this.
       const connection = await this.connections.getStreamingConnection();
+
       const promiseConnection = (connection as mysql.Connection).promise();
       try {
+        await promiseConnection.query(`SET time_zone = '+00:00'`);
         await promiseConnection.query('BEGIN');
         try {
           gtid = await common.readExecutedGtid(promiseConnection);
@@ -258,6 +260,8 @@ AND table_type = 'BASE TABLE';`,
         'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY'
       );
       await promiseConnection.query<mysqlPromise.RowDataPacket[]>('START TRANSACTION');
+      await promiseConnection.query(`SET time_zone = '+00:00'`);
+
       const sourceTables = this.syncRules.getSourceTables();
       await this.storage.startBatch(
         { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true },
@@ -291,38 +295,36 @@ AND table_type = 'BASE TABLE';`,
     logger.info(`Replicating ${table.qualifiedName}`);
     // TODO count rows and log progress at certain batch sizes
 
-    let columns: Map<string, ColumnDescriptor>;
-    return new Promise<void>((resolve, reject) => {
-      // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
-      connection
-        .query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${table.schema}.${table.table}`)
-        .on('error', (err) => {
-          reject(err);
-        })
-        .on('fields', (fields: FieldPacket[]) => {
-          // Map the columns and their types
-          columns = toColumnDescriptors(fields);
-        })
-        .on('result', async (row) => {
-          connection.pause();
-          const record = common.toSQLiteRow(row, columns);
-
-          await batch.save({
-            tag: storage.SaveOperationTag.INSERT,
-            sourceTable: table,
-            before: undefined,
-            beforeReplicaId: undefined,
-            after: record,
-            afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns)
-          });
-          connection.resume();
-          Metrics.getInstance().rows_replicated_total.add(1);
-        })
-        .on('end', async function () {
-          await batch.flush();
-          resolve();
-        });
+    // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
+    const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${escapeMysqlTableName(table)}`);
+    const stream = query.stream();
+
+    let columns: Map<string, ColumnDescriptor> | undefined = undefined;
+    stream.on('fields', (fields: FieldPacket[]) => {
+      // Map the columns and their types
+      columns = toColumnDescriptors(fields);
     });
+
+    for await (let row of stream) {
+      if (this.stopped) {
+        throw new Error('Abort signal received - initial replication interrupted.');
+      }
+
+      if (columns == null) {
+        throw new Error(`No 'fields' event emitted`);
+      }
+
+      const record = common.toSQLiteRow(row, columns!);
+      await batch.save({
+        tag: storage.SaveOperationTag.INSERT,
+        sourceTable: table,
+        before: undefined,
+        beforeReplicaId: undefined,
+        after: record,
+        afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns)
+      });
+    }
+    await batch.flush();
   }
 
   async replicate() {
@@ -350,6 +352,18 @@ AND table_type = 'BASE TABLE';`,
     const initialReplicationCompleted = await this.checkInitialReplicated();
     if (!initialReplicationCompleted) {
       await this.startInitialReplication();
+    } else {
+      // We need to find the existing tables, to populate our table cache.
+      // This is needed for includeSchema to work correctly.
+      const sourceTables = this.syncRules.getSourceTables();
+      await this.storage.startBatch(
+        { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true },
+        async (batch) => {
+          for (let tablePattern of sourceTables) {
+            await this.getQualifiedTableNames(batch, tablePattern);
+          }
+        }
+      );
     }
   }
 
@@ -466,11 +480,36 @@ AND table_type = 'BASE TABLE';`,
             return;
           }
 
-          logger.info(`Reading binlog from: ${binLogPositionState.filename}:${binLogPositionState.offset}`);
+          // Set a heartbeat interval for the Zongji replication connection
+          // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown
+          // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket.
+          await new Promise((resolve, reject) => {
+            zongji.connection.query(
+              // In nanoseconds, 10^9 = 1s
+              'set @master_heartbeat_period=28*1000000000',
+              function (error: any, results: any, fields: any) {
+                if (error) {
+                  reject(error);
+                } else {
+                  resolve(results);
+                }
+              }
+            );
+          });
+          logger.info('Successfully set up replication connection heartbeat...');
 
+          // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat.
+          // The timeout here must be greater than the master_heartbeat_period.
+          const socket = zongji.connection._socket!;
+          socket.setTimeout(60_000, () => {
+            socket.destroy(new Error('Replication connection timeout.'));
+          });
+
+          logger.info(`Reading binlog from: ${binLogPositionState.filename}:${binLogPositionState.offset}`);
           // Only listen for changes to tables in the sync rules
           const includedTables = [...this.tableCache.values()].map((table) => table.table);
           zongji.start({
+            // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive
             includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
             excludeEvents: [],
             includeSchema: { [this.defaultSchema]: includedTables },
@@ -482,7 +521,7 @@ AND table_type = 'BASE TABLE';`,
           // Forever young
           await new Promise<void>((resolve, reject) => {
             zongji.on('error', (error) => {
-              logger.error('Error on Binlog listener:', error);
+              logger.error('Binlog listener error:', error);
               zongji.stop();
               queue.kill();
               reject(error);
@@ -578,7 +617,7 @@ AND table_type = 'BASE TABLE';`,
           beforeReplicaId: beforeUpdated
             ? getUuidReplicaIdentityBson(beforeUpdated, payload.sourceTable.replicaIdColumns)
             : undefined,
-          after: common.toSQLiteRow(payload.data, payload.columns),
+          after: after,
           afterReplicaId: getUuidReplicaIdentityBson(after, payload.sourceTable.replicaIdColumns)
         });
 

package/src/replication/MySQLConnectionManager.ts

@@ -62,7 +62,14 @@ export class MySQLConnectionManager {
    *  @param params
    */
   async query(query: string, params?: any[]): Promise<[RowDataPacket[], FieldPacket[]]> {
-    return this.promisePool.query<RowDataPacket[]>(query, params);
+    let connection: mysqlPromise.PoolConnection | undefined;
+    try {
+      connection = await this.promisePool.getConnection();
+      await connection.query(`SET time_zone = '+00:00'`);
+      return connection.query<RowDataPacket[]>(query, params);
+    } finally {
+      connection?.release();
+    }
   }
 
   /**

package/src/replication/zongji/zongji.d.ts

@@ -1,4 +1,6 @@
 declare module '@powersync/mysql-zongji' {
+  import { Socket } from 'net';
+
   export type ZongjiOptions = {
     host: string;
     user: string;
@@ -108,7 +110,15 @@ declare module '@powersync/mysql-zongji' {
 
   export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent;
 
+  // @vlasky/mysql Connection
+  export interface MySQLConnection {
+    _socket?: Socket;
+    /** There are other forms of this method as well - this is the most basic one. */
+    query(sql: string, callback: (error: any, results: any, fields: any) => void): void;
+  }
+
   export default class ZongJi {
+    connection: MySQLConnection;
     constructor(options: ZongjiOptions);
 
     start(options: StartOptions): void;

package/src/utils/mysql-utils.ts

@@ -3,6 +3,7 @@ import mysql from 'mysql2';
 import mysqlPromise from 'mysql2/promise';
 import * as types from '../types/types.js';
 import { coerce, gte } from 'semver';
+import { SourceTable } from '@powersync/service-core';
 
 export type RetriedQueryOptions = {
   connection: mysqlPromise.Connection;
@@ -82,3 +83,7 @@ export function isVersionAtLeast(version: string, minimumVersion: string): boole
 
   return gte(coercedVersion!, coercedMinimumVersion!, { loose: true });
 }
+
+export function escapeMysqlTableName(table: SourceTable): string {
+  return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.table.replaceAll('`', '``')}\``;
+}