@powersync/service-module-mysql 0.0.0-dev-20241022094219 → 0.0.0-dev-20241023230541

package/src/replication/BinLogStream.ts

@@ -2,15 +2,16 @@ import { logger } from '@powersync/lib-services-framework';
 import * as sync_rules from '@powersync/service-sync-rules';
 import async from 'async';
 
-import { framework, getUuidReplicaIdentityBson, Metrics, storage } from '@powersync/service-core';
-import mysql from 'mysql2';
+import { ColumnDescriptor, framework, getUuidReplicaIdentityBson, Metrics, storage } from '@powersync/service-core';
+import mysql, { FieldPacket } from 'mysql2';
 
-import { BinLogEvent } from '@powersync/mysql-zongji';
+import { BinLogEvent, TableMapEntry } from '@powersync/mysql-zongji';
 import * as common from '../common/common-index.js';
 import * as zongji_utils from './zongji/zongji-utils.js';
 import { MySQLConnectionManager } from './MySQLConnectionManager.js';
-import { ReplicatedGTID } from '../common/common-index.js';
+import { isBinlogStillAvailable, ReplicatedGTID } from '../common/common-index.js';
 import mysqlPromise from 'mysql2/promise';
+import { MySQLTypesMap } from '../utils/mysql_utils.js';
 
 export interface BinLogStreamOptions {
   connections: MySQLConnectionManager;
@@ -30,6 +31,7 @@ interface WriteChangePayload {
   database: string;
   table: string;
   sourceTable: storage.SourceTable;
+  columns: Map<string, ColumnDescriptor>;
 }
 
 export type Data = Record<string, any>;
@@ -211,10 +213,23 @@ AND table_type = 'BASE TABLE';`,
    */
   protected async checkInitialReplicated(): Promise<boolean> {
     const status = await this.storage.getStatus();
+    const lastKnowGTID = status.checkpoint_lsn ? common.ReplicatedGTID.fromSerialized(status.checkpoint_lsn) : null;
     if (status.snapshot_done && status.checkpoint_lsn) {
-      logger.info(`Initial replication already done. MySQL appears healthy`);
+      logger.info(`Initial replication already done.`);
+
+      if (lastKnowGTID) {
+        // Check if the binlog is still available. If it isn't we need to snapshot again.
+        const connection = await this.connections.getConnection();
+        try {
+          return await isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
+        } finally {
+          connection.release();
+        }
+      }
+
       return true;
     }
+
     return false;
   }
 
@@ -270,17 +285,24 @@ AND table_type = 'BASE TABLE';`,
     logger.info(`Replicating ${table.qualifiedName}`);
     // TODO count rows and log progress at certain batch sizes
 
+    const columns = new Map<string, ColumnDescriptor>();
     return new Promise<void>((resolve, reject) => {
       // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
       connection
         .query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${table.schema}.${table.table}`)
-        .stream()
         .on('error', (err) => {
           reject(err);
         })
-        .on('data', async (row) => {
+        .on('fields', (fields: FieldPacket[]) => {
+          // Map the columns and their types
+          fields.forEach((field) => {
+            const columnType = MySQLTypesMap[field.type as number];
+            columns.set(field.name, { name: field.name, type: columnType, typeId: field.type });
+          });
+        })
+        .on('result', async (row) => {
           connection.pause();
-          const record = common.toSQLiteRow(row);
+          const record = common.toSQLiteRow(row, columns);
 
           await batch.save({
             tag: storage.SaveOperationTag.INSERT,
@@ -291,7 +313,6 @@ AND table_type = 'BASE TABLE';`,
             afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns)
           });
           connection.resume();
-          // TODO: These metrics can probably be reported in batches
           Metrics.getInstance().rows_replicated_total.add(1);
         })
         .on('end', async function () {
@@ -382,19 +403,11 @@ AND table_type = 'BASE TABLE';`,
                 binLogPositionState.offset = evt.position;
                 break;
               case zongji_utils.eventIsWriteMutation(evt):
-                // TODO, can multiple tables be present?
                 const writeTableInfo = evt.tableMap[evt.tableId];
                 await this.writeChanges(batch, {
                   type: storage.SaveOperationTag.INSERT,
                   data: evt.rows,
-                  database: writeTableInfo.parentSchema,
-                  table: writeTableInfo.tableName,
-                  sourceTable: this.getTable(
-                    getMysqlRelId({
-                      schema: writeTableInfo.parentSchema,
-                      name: writeTableInfo.tableName
-                    })
-                  )
+                  tableEntry: writeTableInfo
                 });
                 break;
               case zongji_utils.eventIsUpdateMutation(evt):
@@ -403,31 +416,15 @@ AND table_type = 'BASE TABLE';`,
                   type: storage.SaveOperationTag.UPDATE,
                   data: evt.rows.map((row) => row.after),
                   previous_data: evt.rows.map((row) => row.before),
-                  database: updateTableInfo.parentSchema,
-                  table: updateTableInfo.tableName,
-                  sourceTable: this.getTable(
-                    getMysqlRelId({
-                      schema: updateTableInfo.parentSchema,
-                      name: updateTableInfo.tableName
-                    })
-                  )
+                  tableEntry: updateTableInfo
                 });
                 break;
               case zongji_utils.eventIsDeleteMutation(evt):
-                // TODO, can multiple tables be present?
                 const deleteTableInfo = evt.tableMap[evt.tableId];
                 await this.writeChanges(batch, {
                   type: storage.SaveOperationTag.DELETE,
                   data: evt.rows,
-                  database: deleteTableInfo.parentSchema,
-                  table: deleteTableInfo.tableName,
-                  // TODO cleanup
-                  sourceTable: this.getTable(
-                    getMysqlRelId({
-                      schema: deleteTableInfo.parentSchema,
-                      name: deleteTableInfo.tableName
-                    })
-                  )
+                  tableEntry: deleteTableInfo
                 });
                 break;
               case zongji_utils.eventIsXid(evt):
@@ -503,14 +500,26 @@ AND table_type = 'BASE TABLE';`,
       type: storage.SaveOperationTag;
       data: Data[];
       previous_data?: Data[];
-      database: string;
-      table: string;
-      sourceTable: storage.SourceTable;
+      tableEntry: TableMapEntry;
     }
   ): Promise<storage.FlushedResult | null> {
+    const columns = new Map<string, ColumnDescriptor>();
+    msg.tableEntry.columns.forEach((column) => {
+      columns.set(column.name, { name: column.name, typeId: column.type });
+    });
+
     for (const [index, row] of msg.data.entries()) {
       await this.writeChange(batch, {
-        ...msg,
+        type: msg.type,
+        database: msg.tableEntry.parentSchema,
+        sourceTable: this.getTable(
+          getMysqlRelId({
+            schema: msg.tableEntry.parentSchema,
+            name: msg.tableEntry.tableName
+          })
+        ),
+        table: msg.tableEntry.tableName,
+        columns: columns,
         data: row,
         previous_data: msg.previous_data?.[index]
       });
@@ -525,7 +534,7 @@ AND table_type = 'BASE TABLE';`,
     switch (payload.type) {
       case storage.SaveOperationTag.INSERT:
         Metrics.getInstance().rows_replicated_total.add(1);
-        const record = common.toSQLiteRow(payload.data);
+        const record = common.toSQLiteRow(payload.data, payload.columns);
         return await batch.save({
           tag: storage.SaveOperationTag.INSERT,
           sourceTable: payload.sourceTable,

package/src/replication/zongji/zongji.d.ts

@@ -4,6 +4,7 @@ declare module '@powersync/mysql-zongji' {
     user: string;
     password: string;
     dateStrings?: boolean;
+    timeZone?: string;
   };
 
   interface DatabaseFilter {
@@ -31,6 +32,11 @@ declare module '@powersync/mysql-zongji' {
      * BinLog position offset to start reading events from in file specified
      */
     position?: number;
+
+    /**
+     *  Unique server ID for this replication client.
+     */
+    serverId?: number;
   };
 
   export type ColumnSchema = {
@@ -49,10 +55,10 @@ declare module '@powersync/mysql-zongji' {
   };
 
   export type TableMapEntry = {
-    columnSchemas: Array<ColumnSchema>;
+    columnSchemas: ColumnSchema[];
     parentSchema: string;
     tableName: string;
-    columns: Array<ColumnDefinition>;
+    columns: ColumnDefinition[];
   };
 
   export type BaseBinLogEvent = {
@@ -90,14 +96,14 @@ declare module '@powersync/mysql-zongji' {
     tableId: number;
     numberOfColumns: number;
     tableMap: Record<string, TableMapEntry>;
-    rows: Array<Record<string, any>>;
+    rows: Record<string, any>[];
   };
 
   export type BinLogUpdateEvent = Omit<BinLogMutationEvent, 'rows'> & {
-    rows: Array<{
+    rows: {
       before: Record<string, any>;
       after: Record<string, any>;
-    }>;
+    }[];
   };
 
   export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent;

package/src/types/types.ts

@@ -14,6 +14,7 @@ export interface NormalizedMySQLConnectionConfig {
 
   username: string;
   password: string;
+  server_id: number;
 
   cacert?: string;
   client_certificate?: string;
@@ -29,6 +30,7 @@ export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.a
     username: t.string.optional(),
     password: t.string.optional(),
     database: t.string.optional(),
+    server_id: t.number.optional(),
 
     cacert: t.string.optional(),
     client_certificate: t.string.optional(),
@@ -97,6 +99,8 @@ export function normalizeConnectionConfig(options: MySQLConnectionConfig): Norma
     database,
 
     username,
-    password
+    password,
+
+    server_id: options.server_id ?? 1
   };
 }

package/src/utils/mysql_utils.ts

@@ -3,6 +3,11 @@ import mysql from 'mysql2';
 import mysqlPromise from 'mysql2/promise';
 import * as types from '../types/types.js';
 
+export const MySQLTypesMap: { [key: number]: string } = {};
+for (const [name, code] of Object.entries(mysql.Types)) {
+  MySQLTypesMap[code as number] = name;
+}
+
 export type RetriedQueryOptions = {
   connection: mysqlPromise.Connection;
   query: string;
@@ -35,13 +40,14 @@ export function createPool(config: types.NormalizedMySQLConnectionConfig, option
     cert: config.client_certificate
   };
   const hasSSLOptions = Object.values(sslOptions).some((v) => !!v);
-  // TODO confirm if default options are fine for Powersync use case
   return mysql.createPool({
     host: config.hostname,
     user: config.username,
     password: config.password,
     database: config.database,
     ssl: hasSSLOptions ? sslOptions : undefined,
+    supportBigNumbers: true,
+    timezone: 'Z', // Ensure no auto timezone manipulation of the dates occur
     ...(options || {})
   });
 }