@powersync/service-module-mysql 0.0.0-dev-20241101083236 → 0.0.0-dev-20241107065634

package/src/common/mysql-to-sqlite.ts

@@ -2,16 +2,155 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import { ExpressionType } from '@powersync/service-sync-rules';
 import { ColumnDescriptor } from '@powersync/service-core';
 import mysql from 'mysql2';
+import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
+import { ColumnDefinition, TableMapEntry } from '@powersync/mysql-zongji';
 
-export function toSQLiteRow(row: Record<string, any>, columns?: Map<string, ColumnDescriptor>): sync_rules.SqliteRow {
+export enum ADDITIONAL_MYSQL_TYPES {
+  DATETIME2 = 18,
+  TIMESTAMP2 = 17,
+  BINARY = 100,
+  VARBINARY = 101,
+  TEXT = 102
+}
+
+export const MySQLTypesMap: { [key: number]: string } = {};
+for (const [name, code] of Object.entries(mysql.Types)) {
+  MySQLTypesMap[code as number] = name;
+}
+for (const [name, code] of Object.entries(ADDITIONAL_MYSQL_TYPES)) {
+  MySQLTypesMap[code as number] = name;
+}
+
+export function toColumnDescriptors(columns: mysql.FieldPacket[]): Map<string, ColumnDescriptor>;
+export function toColumnDescriptors(tableMap: TableMapEntry): Map<string, ColumnDescriptor>;
+
+export function toColumnDescriptors(columns: mysql.FieldPacket[] | TableMapEntry): Map<string, ColumnDescriptor> {
+  const columnMap = new Map<string, ColumnDescriptor>();
+  if (Array.isArray(columns)) {
+    for (const column of columns) {
+      columnMap.set(column.name, toColumnDescriptorFromFieldPacket(column));
+    }
+  } else {
+    for (const column of columns.columns) {
+      columnMap.set(column.name, toColumnDescriptorFromDefinition(column));
+    }
+  }
+
+  return columnMap;
+}
+
+export function toColumnDescriptorFromFieldPacket(column: mysql.FieldPacket): ColumnDescriptor {
+  let typeId = column.type!;
+  const BINARY_FLAG = 128;
+  const MYSQL_ENUM_FLAG = 256;
+  const MYSQL_SET_FLAG = 2048;
+
+  switch (column.type) {
+    case mysql.Types.STRING:
+      if (((column.flags as number) & BINARY_FLAG) !== 0) {
+        typeId = ADDITIONAL_MYSQL_TYPES.BINARY;
+      } else if (((column.flags as number) & MYSQL_ENUM_FLAG) !== 0) {
+        typeId = mysql.Types.ENUM;
+      } else if (((column.flags as number) & MYSQL_SET_FLAG) !== 0) {
+        typeId = mysql.Types.SET;
+      }
+      break;
+
+    case mysql.Types.VAR_STRING:
+      typeId = ((column.flags as number) & BINARY_FLAG) !== 0 ? ADDITIONAL_MYSQL_TYPES.VARBINARY : column.type;
+      break;
+    case mysql.Types.BLOB:
+      typeId = ((column.flags as number) & BINARY_FLAG) === 0 ? ADDITIONAL_MYSQL_TYPES.TEXT : column.type;
+      break;
+  }
+
+  const columnType = MySQLTypesMap[typeId];
+
+  return {
+    name: column.name,
+    type: columnType,
+    typeId: typeId
+  };
+}
+
+export function toColumnDescriptorFromDefinition(column: ColumnDefinition): ColumnDescriptor {
+  let typeId = column.type;
+
+  switch (column.type) {
+    case mysql.Types.STRING:
+      typeId = !column.charset ? ADDITIONAL_MYSQL_TYPES.BINARY : column.type;
+      break;
+    case mysql.Types.VAR_STRING:
+    case mysql.Types.VARCHAR:
+      typeId = !column.charset ? ADDITIONAL_MYSQL_TYPES.VARBINARY : column.type;
+      break;
+    case mysql.Types.BLOB:
+      typeId = column.charset ? ADDITIONAL_MYSQL_TYPES.TEXT : column.type;
+      break;
+  }
+
+  const columnType = MySQLTypesMap[typeId];
+
+  return {
+    name: column.name,
+    type: columnType,
+    typeId: typeId
+  };
+}
+
+export function toSQLiteRow(row: Record<string, any>, columns: Map<string, ColumnDescriptor>): sync_rules.SqliteRow {
   for (let key in row) {
-    if (row[key] instanceof Date) {
-      const column = columns?.get(key);
-      if (column?.typeId == mysql.Types.DATE) {
-        // Only parse the date part
-        row[key] = row[key].toISOString().split('T')[0];
-      } else {
-        row[key] = row[key].toISOString();
+    // We are very much expecting the column to be there
+    const column = columns.get(key)!;
+
+    if (row[key] !== null) {
+      switch (column.typeId) {
+        case mysql.Types.DATE:
+          // Only parse the date part
+          row[key] = row[key].toISOString().split('T')[0];
+          break;
+        case mysql.Types.DATETIME:
+        case ADDITIONAL_MYSQL_TYPES.DATETIME2:
+        case mysql.Types.TIMESTAMP:
+        case ADDITIONAL_MYSQL_TYPES.TIMESTAMP2:
+          row[key] = row[key].toISOString();
+          break;
+        case mysql.Types.JSON:
+          if (typeof row[key] === 'string') {
+            row[key] = new JsonContainer(row[key]);
+          }
+          break;
+        case mysql.Types.BIT:
+        case mysql.Types.BLOB:
+        case mysql.Types.TINY_BLOB:
+        case mysql.Types.MEDIUM_BLOB:
+        case mysql.Types.LONG_BLOB:
+        case ADDITIONAL_MYSQL_TYPES.BINARY:
+        case ADDITIONAL_MYSQL_TYPES.VARBINARY:
+          row[key] = new Uint8Array(Object.values(row[key]));
+          break;
+        case mysql.Types.LONGLONG:
+          if (typeof row[key] === 'string') {
+            row[key] = BigInt(row[key]);
+          } else if (typeof row[key] === 'number') {
+            // Zongji returns BIGINT as a number when it can be represented as a number
+            row[key] = BigInt(row[key]);
+          }
+          break;
+        case mysql.Types.TINY:
+        case mysql.Types.SHORT:
+        case mysql.Types.LONG:
+        case mysql.Types.INT24:
+          // Handle all integer values a BigInt
+          if (typeof row[key] === 'number') {
+            row[key] = BigInt(row[key]);
+          }
+          break;
+        case mysql.Types.SET:
+          // Convert to JSON array from string
+          const values = row[key].split(',');
+          row[key] = JSONBig.stringify(values);
+          break;
       }
     }
   }

package/src/common/read-executed-gtid.ts

@@ -4,7 +4,6 @@ import { gte } from 'semver';
 
 import { ReplicatedGTID } from './ReplicatedGTID.js';
 import { getMySQLVersion } from './check-source-configuration.js';
-import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Gets the current master HEAD GTID
@@ -33,8 +32,6 @@ export async function readExecutedGtid(connection: mysqlPromise.Connection): Pro
     offset: parseInt(binlogStatus.Position)
   };
 
-  logger.info('Succesfully read executed GTID', { position });
-
   return new ReplicatedGTID({
     // The head always points to the next position to start replication from
     position,

package/src/replication/BinLogStream.ts

@@ -9,9 +9,9 @@ import { BinLogEvent, StartOptions, TableMapEntry } from '@powersync/mysql-zongj
 import * as common from '../common/common-index.js';
 import * as zongji_utils from './zongji/zongji-utils.js';
 import { MySQLConnectionManager } from './MySQLConnectionManager.js';
-import { isBinlogStillAvailable, ReplicatedGTID } from '../common/common-index.js';
+import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js';
 import mysqlPromise from 'mysql2/promise';
-import { MySQLTypesMap } from '../utils/mysql_utils.js';
+import { createRandomServerId } from '../utils/mysql_utils.js';
 
 export interface BinLogStreamOptions {
   connections: MySQLConnectionManager;
@@ -221,7 +221,13 @@ AND table_type = 'BASE TABLE';`,
         // Check if the binlog is still available. If it isn't we need to snapshot again.
         const connection = await this.connections.getConnection();
         try {
-          return await isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
+          const isAvailable = await isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
+          if (!isAvailable) {
+            logger.info(
+              `Binlog file ${lastKnowGTID.position.filename} is no longer available, starting initial replication again.`
+            );
+          }
+          return isAvailable;
         } finally {
           connection.release();
         }
@@ -245,7 +251,7 @@ AND table_type = 'BASE TABLE';`,
     const connection = await this.connections.getStreamingConnection();
     const promiseConnection = (connection as mysql.Connection).promise();
     const headGTID = await common.readExecutedGtid(promiseConnection);
-    logger.info(`Using snapshot checkpoint GTID:: '${headGTID}'`);
+    logger.info(`Using snapshot checkpoint GTID: '${headGTID}'`);
     try {
       logger.info(`Starting initial replication`);
       await promiseConnection.query<mysqlPromise.RowDataPacket[]>(
@@ -285,7 +291,7 @@ AND table_type = 'BASE TABLE';`,
     logger.info(`Replicating ${table.qualifiedName}`);
     // TODO count rows and log progress at certain batch sizes
 
-    const columns = new Map<string, ColumnDescriptor>();
+    let columns: Map<string, ColumnDescriptor>;
     return new Promise<void>((resolve, reject) => {
       // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
       connection
@@ -295,10 +301,7 @@ AND table_type = 'BASE TABLE';`,
         })
         .on('fields', (fields: FieldPacket[]) => {
           // Map the columns and their types
-          fields.forEach((field) => {
-            const columnType = MySQLTypesMap[field.type as number];
-            columns.set(field.name, { name: field.name, type: columnType, typeId: field.type });
-          });
+          columns = toColumnDescriptors(fields);
         })
         .on('result', async (row) => {
           connection.pause();
@@ -363,10 +366,14 @@ AND table_type = 'BASE TABLE';`,
   async streamChanges() {
     // Auto-activate as soon as initial replication is done
     await this.storage.autoActivate();
+    const serverId = createRandomServerId(this.storage.group_id);
+    logger.info(`Starting replication. Created replica client with serverId:${serverId}`);
 
     const connection = await this.connections.getConnection();
     const { checkpoint_lsn } = await this.storage.getStatus();
-    logger.info(`Last known LSN from storage: ${checkpoint_lsn}`);
+    if (checkpoint_lsn) {
+      logger.info(`Existing checkpoint found: ${checkpoint_lsn}`);
+    }
 
     const fromGTID = checkpoint_lsn
       ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn)
@@ -447,7 +454,7 @@ AND table_type = 'BASE TABLE';`,
 
           zongji.on('binlog', (evt: BinLogEvent) => {
             if (!this.stopped) {
-              logger.info(`Pushing Binlog event ${evt.getEventName()}`);
+              logger.info(`Received Binlog event:${evt.getEventName()}`);
               queue.push(evt);
             } else {
               logger.info(`Replication is busy stopping, ignoring event ${evt.getEventName()}`);
@@ -458,16 +465,18 @@ AND table_type = 'BASE TABLE';`,
             // Powersync is shutting down, don't start replicating
             return;
           }
+
+          logger.info(`Reading binlog from: ${binLogPositionState.filename}:${binLogPositionState.offset}`);
+
           // Only listen for changes to tables in the sync rules
           const includedTables = [...this.tableCache.values()].map((table) => table.table);
-          logger.info(`Starting replication from ${binLogPositionState.filename}:${binLogPositionState.offset}`);
           zongji.start({
             includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
             excludeEvents: [],
             includeSchema: { [this.defaultSchema]: includedTables },
             filename: binLogPositionState.filename,
             position: binLogPositionState.offset,
-            serverId: this.storage.group_id
+            serverId: serverId
           } satisfies StartOptions);
 
           // Forever young
@@ -516,10 +525,7 @@ AND table_type = 'BASE TABLE';`,
       tableEntry: TableMapEntry;
     }
   ): Promise<storage.FlushedResult | null> {
-    const columns = new Map<string, ColumnDescriptor>();
-    msg.tableEntry.columns.forEach((column) => {
-      columns.set(column.name, { name: column.name, typeId: column.type });
-    });
+    const columns = toColumnDescriptors(msg.tableEntry);
 
     for (const [index, row] of msg.data.entries()) {
       await this.writeChange(batch, {
@@ -560,8 +566,10 @@ AND table_type = 'BASE TABLE';`,
         Metrics.getInstance().rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
-        const beforeUpdated = payload.previous_data ? common.toSQLiteRow(payload.previous_data) : undefined;
-        const after = common.toSQLiteRow(payload.data);
+        const beforeUpdated = payload.previous_data
+          ? common.toSQLiteRow(payload.previous_data, payload.columns)
+          : undefined;
+        const after = common.toSQLiteRow(payload.data, payload.columns);
 
         return await batch.save({
           tag: storage.SaveOperationTag.UPDATE,
@@ -570,13 +578,13 @@ AND table_type = 'BASE TABLE';`,
           beforeReplicaId: beforeUpdated
             ? getUuidReplicaIdentityBson(beforeUpdated, payload.sourceTable.replicaIdColumns)
             : undefined,
-          after: common.toSQLiteRow(payload.data),
+          after: common.toSQLiteRow(payload.data, payload.columns),
           afterReplicaId: getUuidReplicaIdentityBson(after, payload.sourceTable.replicaIdColumns)
         });
 
       case storage.SaveOperationTag.DELETE:
         Metrics.getInstance().rows_replicated_total.add(1);
-        const beforeDeleted = common.toSQLiteRow(payload.data);
+        const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns);
 
         return await batch.save({
           tag: storage.SaveOperationTag.DELETE,

package/src/replication/MySQLConnectionManager.ts

@@ -1,6 +1,6 @@
 import { NormalizedMySQLConnectionConfig } from '../types/types.js';
 import mysqlPromise from 'mysql2/promise';
-import mysql, { RowDataPacket } from 'mysql2';
+import mysql, { FieldPacket, RowDataPacket } from 'mysql2';
 import * as mysql_utils from '../utils/mysql_utils.js';
 import ZongJi from '@powersync/mysql-zongji';
 import { logger } from '@powersync/lib-services-framework';
@@ -61,7 +61,7 @@ export class MySQLConnectionManager {
    *  @param query
    *  @param params
    */
-  async query(query: string, params?: any[]) {
+  async query(query: string, params?: any[]): Promise<[RowDataPacket[], FieldPacket[]]> {
     return this.promisePool.query<RowDataPacket[]>(query, params);
   }
 

package/src/utils/mysql_utils.ts

@@ -3,11 +3,6 @@ import mysql from 'mysql2';
 import mysqlPromise from 'mysql2/promise';
 import * as types from '../types/types.js';
 
-export const MySQLTypesMap: { [key: number]: string } = {};
-for (const [name, code] of Object.entries(mysql.Types)) {
-  MySQLTypesMap[code as number] = name;
-}
-
 export type RetriedQueryOptions = {
   connection: mysqlPromise.Connection;
   query: string;
@@ -47,7 +42,21 @@ export function createPool(config: types.NormalizedMySQLConnectionConfig, option
     database: config.database,
     ssl: hasSSLOptions ? sslOptions : undefined,
     supportBigNumbers: true,
+    decimalNumbers: true,
     timezone: 'Z', // Ensure no auto timezone manipulation of the dates occur
+    jsonStrings: true, // Return JSON columns as strings
     ...(options || {})
   });
 }
+
+/**
+ *  Return a random server id for a given sync rule id.
+ *  Expected format is: <syncRuleId>00<random number>
+ *  The max value for server id in MySQL is 2^32 - 1.
+ *  We use the GTID format to keep track of our position in the binlog, no state is kept by the MySQL server, therefore
+ *  it is ok to use a randomised server id every time.
+ *  @param syncRuleId
+ */
+export function createRandomServerId(syncRuleId: number): number {
+  return Number.parseInt(`${syncRuleId}00${Math.floor(Math.random() * 10000)}`);
+}