@powersync/service-module-mysql 0.0.0-dev-20241015210820

package/src/api/MySQLRouteAPIAdapter.ts

@@ -0,0 +1,357 @@
+import { api, ParseSyncRulesOptions, storage } from '@powersync/service-core';
+
+import * as sync_rules from '@powersync/service-sync-rules';
+import * as service_types from '@powersync/service-types';
+import mysql from 'mysql2/promise';
+import * as common from '../common/common-index.js';
+import * as mysql_utils from '../utils/mysql_utils.js';
+import * as types from '../types/types.js';
+import { toExpressionTypeFromMySQLType } from '../common/common-index.js';
+
+type SchemaResult = {
+  schema_name: string;
+  table_name: string;
+  columns: Array<{ data_type: string; column_name: string }>;
+};
+
+export class MySQLRouteAPIAdapter implements api.RouteAPI {
+  protected pool: mysql.Pool;
+
+  constructor(protected config: types.ResolvedConnectionConfig) {
+    this.pool = mysql_utils.createPool(config).promise();
+  }
+
+  async shutdown(): Promise<void> {
+    return this.pool.end();
+  }
+
+  async getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig> {
+    return this.config;
+  }
+
+  getParseSyncRulesOptions(): ParseSyncRulesOptions {
+    return {
+      // In MySQL Schema and Database are the same thing. There is no default database
+      defaultSchema: this.config.database
+    };
+  }
+
+  async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
+    const base = {
+      id: this.config.id,
+      uri: `mysql://${this.config.hostname}:${this.config.port}/${this.config.database}`
+    };
+    try {
+      await this.retriedQuery({
+        query: `SELECT 'PowerSync connection test'`
+      });
+    } catch (e) {
+      return {
+        ...base,
+        connected: false,
+        errors: [{ level: 'fatal', message: `${e.code} - message: ${e.message}` }]
+      };
+    }
+    const connection = await this.pool.getConnection();
+    try {
+      const errors = await common.checkSourceConfiguration(connection);
+      if (errors.length) {
+        return {
+          ...base,
+          connected: true,
+          errors: errors.map((e) => ({ level: 'fatal', message: e }))
+        };
+      }
+    } catch (e) {
+      return {
+        ...base,
+        connected: true,
+        errors: [{ level: 'fatal', message: e.message }]
+      };
+    } finally {
+      connection.release();
+    }
+    return {
+      ...base,
+      connected: true,
+      errors: []
+    };
+  }
+
+  async executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse> {
+    if (!this.config.debug_api) {
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: 'SQL querying is not enabled'
+      });
+    }
+    try {
+      const [results, fields] = await this.pool.query<mysql.RowDataPacket[]>(query, params);
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        success: true,
+        results: {
+          columns: fields.map((c) => c.name),
+          rows: results.map((row) => {
+            /**
+             * Row will be in the format:
+             * @rows: [ { test: 2 } ]
+             */
+            return fields.map((c) => {
+              const value = row[c.name];
+              const sqlValue = sync_rules.toSyncRulesValue(value);
+              if (typeof sqlValue == 'bigint') {
+                return Number(value);
+              } else if (value instanceof Date) {
+                return value.toISOString();
+              } else if (sync_rules.isJsonValue(sqlValue)) {
+                return sqlValue;
+              } else {
+                return null;
+              }
+            });
+          })
+        }
+      });
+    } catch (e) {
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: e.message
+      });
+    }
+  }
+
+  async getDebugTablesInfo(
+    tablePatterns: sync_rules.TablePattern[],
+    sqlSyncRules: sync_rules.SqlSyncRules
+  ): Promise<api.PatternResult[]> {
+    let result: api.PatternResult[] = [];
+
+    for (let tablePattern of tablePatterns) {
+      const schema = tablePattern.schema;
+      let patternResult: api.PatternResult = {
+        schema: schema,
+        pattern: tablePattern.tablePattern,
+        wildcard: tablePattern.isWildcard
+      };
+      result.push(patternResult);
+
+      if (tablePattern.isWildcard) {
+        patternResult.tables = [];
+        const prefix = tablePattern.tablePrefix;
+
+        const [results] = await this.pool.query<mysql.RowDataPacket[]>(
+          `SELECT
+            TABLE_NAME AS table_name
+           FROM 
+            INFORMATION_SCHEMA.TABLES
+           WHERE 
+            TABLE_SCHEMA = ?
+            AND TABLE_NAME LIKE ?`,
+          [schema, tablePattern.tablePattern]
+        );
+
+        for (let row of results) {
+          const name = row.table_name as string;
+
+          if (!name.startsWith(prefix)) {
+            continue;
+          }
+
+          const details = await this.getDebugTableInfo(tablePattern, name, sqlSyncRules);
+          patternResult.tables.push(details);
+        }
+      } else {
+        const [results] = await this.pool.query<mysql.RowDataPacket[]>(
+          `SELECT
+            TABLE_NAME AS table_name
+           FROM 
+            INFORMATION_SCHEMA.TABLES
+           WHERE 
+            TABLE_SCHEMA = ?
+            AND TABLE_NAME = ?`,
+          [tablePattern.schema, tablePattern.tablePattern]
+        );
+
+        if (results.length == 0) {
+          // Table not found
+          patternResult.table = await this.getDebugTableInfo(tablePattern, tablePattern.name, sqlSyncRules);
+        } else {
+          const row = results[0];
+          patternResult.table = await this.getDebugTableInfo(tablePattern, row.table_name, sqlSyncRules);
+        }
+      }
+    }
+
+    return result;
+  }
+
+  protected async getDebugTableInfo(
+    tablePattern: sync_rules.TablePattern,
+    tableName: string,
+    syncRules: sync_rules.SqlSyncRules
+  ): Promise<service_types.TableInfo> {
+    const { schema } = tablePattern;
+
+    let idColumnsResult: common.ReplicationIdentityColumnsResult | null = null;
+    let idColumnsError: service_types.ReplicationError | null = null;
+    let connection: mysql.PoolConnection | null = null;
+    try {
+      connection = await this.pool.getConnection();
+      idColumnsResult = await common.getReplicationIdentityColumns({
+        connection: connection,
+        schema,
+        table_name: tableName
+      });
+    } catch (ex) {
+      idColumnsError = { level: 'fatal', message: ex.message };
+    } finally {
+      connection?.release();
+    }
+
+    const idColumns = idColumnsResult?.columns ?? [];
+    const sourceTable = new storage.SourceTable(0, this.config.tag, tableName, schema, tableName, idColumns, true);
+    const syncData = syncRules.tableSyncsData(sourceTable);
+    const syncParameters = syncRules.tableSyncsParameters(sourceTable);
+
+    if (idColumns.length == 0 && idColumnsError == null) {
+      let message = `No replication id found for ${sourceTable.qualifiedName}. Replica identity: ${idColumnsResult?.identity}.`;
+      if (idColumnsResult?.identity == 'default') {
+        message += ' Configure a primary key on the table.';
+      }
+      idColumnsError = { level: 'fatal', message };
+    }
+
+    let selectError: service_types.ReplicationError | null = null;
+    try {
+      await this.retriedQuery({
+        query: `SELECT * FROM ${sourceTable.table} LIMIT 1`
+      });
+    } catch (e) {
+      selectError = { level: 'fatal', message: e.message };
+    }
+
+    return {
+      schema: schema,
+      name: tableName,
+      pattern: tablePattern.isWildcard ? tablePattern.tablePattern : undefined,
+      replication_id: idColumns.map((c) => c.name),
+      data_queries: syncData,
+      parameter_queries: syncParameters,
+      errors: [idColumnsError, selectError].filter((error) => error != null) as service_types.ReplicationError[]
+    };
+  }
+
+  async getReplicationLag(options: api.ReplicationLagOptions): Promise<number | undefined> {
+    const { bucketStorage } = options;
+    const lastCheckpoint = await bucketStorage.getCheckpoint();
+
+    const current = lastCheckpoint.lsn
+      ? common.ReplicatedGTID.fromSerialized(lastCheckpoint.lsn)
+      : common.ReplicatedGTID.ZERO;
+
+    const connection = await this.pool.getConnection();
+    const head = await common.readExecutedGtid(connection);
+    const lag = await current.distanceTo(connection, head);
+    connection.release();
+    if (lag == null) {
+      throw new Error(`Could not determine replication lag`);
+    }
+
+    return lag;
+  }
+
+  async getReplicationHead(): Promise<string> {
+    const connection = await this.pool.getConnection();
+    const result = await common.readExecutedGtid(connection);
+    connection.release();
+    return result.comparable;
+  }
+
+  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
+    const [results] = await this.retriedQuery({
+      query: `
+        SELECT 
+          tbl.schema_name,
+          tbl.table_name,
+          tbl.quoted_name,
+          JSON_ARRAYAGG(JSON_OBJECT('column_name', a.column_name, 'data_type', a.data_type)) AS columns
+        FROM
+          (
+            SELECT 
+              TABLE_SCHEMA AS schema_name,
+              TABLE_NAME AS table_name,
+              CONCAT('\`', TABLE_SCHEMA, '\`.\`', TABLE_NAME, '\`') AS quoted_name
+            FROM 
+              INFORMATION_SCHEMA.TABLES
+            WHERE 
+              TABLE_TYPE = 'BASE TABLE'
+              AND TABLE_SCHEMA NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')
+          ) AS tbl
+          LEFT JOIN 
+            (
+              SELECT 
+                TABLE_SCHEMA AS schema_name,
+                TABLE_NAME AS table_name,
+                COLUMN_NAME AS column_name,
+                COLUMN_TYPE AS data_type
+              FROM 
+                INFORMATION_SCHEMA.COLUMNS
+            ) AS a 
+            ON 
+              tbl.schema_name = a.schema_name 
+              AND tbl.table_name = a.table_name
+        GROUP BY 
+          tbl.schema_name, tbl.table_name, tbl.quoted_name;
+      `
+    });
+
+    /**
+     * Reduces the SQL results into a Record of {@link DatabaseSchema}
+     * then returns the values as an array.
+     */
+
+    return Object.values(
+      (results as SchemaResult[]).reduce((hash: Record<string, service_types.DatabaseSchema>, result) => {
+        const schema =
+          hash[result.schema_name] ||
+          (hash[result.schema_name] = {
+            name: result.schema_name,
+            tables: []
+          });
+
+        schema.tables.push({
+          name: result.table_name,
+          columns: result.columns.map((column) => ({
+            name: column.column_name,
+            type: column.data_type,
+            sqlite_type: toExpressionTypeFromMySQLType(column.data_type).typeFlags,
+            internal_type: column.data_type,
+            pg_type: column.data_type
+          }))
+        });
+
+        return hash;
+      }, {})
+    );
+  }
+
+  protected async retriedQuery(options: { query: string; params?: any[] }) {
+    const connection = await this.pool.getConnection();
+
+    return mysql_utils
+      .retriedQuery({
+        connection: connection,
+        query: options.query,
+        params: options.params
+      })
+      .finally(() => connection.release());
+  }
+}

package/src/common/ReplicatedGTID.ts

@@ -0,0 +1,158 @@
+import mysql from 'mysql2/promise';
+import * as uuid from 'uuid';
+import * as mysql_utils from '../utils/mysql_utils.js';
+
+export type BinLogPosition = {
+  filename: string;
+  offset: number;
+};
+
+export type ReplicatedGTIDSpecification = {
+  raw_gtid: string;
+  /**
+   * The (end) position in a BinLog file where this transaction has been replicated in.
+   */
+  position: BinLogPosition;
+};
+
+export type BinLogGTIDFormat = {
+  server_id: Buffer;
+  transaction_range: number;
+};
+
+export type BinLogGTIDEvent = {
+  raw_gtid: BinLogGTIDFormat;
+  position: BinLogPosition;
+};
+
+/**
+ * A wrapper around the MySQL GTID value.
+ * This adds and tracks additional metadata such as the BinLog filename
+ * and position where this GTID could be located.
+ */
+export class ReplicatedGTID {
+  static fromSerialized(comparable: string): ReplicatedGTID {
+    return new ReplicatedGTID(ReplicatedGTID.deserialize(comparable));
+  }
+
+  private static deserialize(comparable: string): ReplicatedGTIDSpecification {
+    const components = comparable.split('|');
+    if (components.length < 3) {
+      throw new Error(`Invalid serialized GTID: ${comparable}`);
+    }
+
+    return {
+      raw_gtid: components[1],
+      position: {
+        filename: components[2],
+        offset: parseInt(components[3])
+      } satisfies BinLogPosition
+    };
+  }
+
+  static fromBinLogEvent(event: BinLogGTIDEvent) {
+    const { raw_gtid, position } = event;
+    const stringGTID = `${uuid.stringify(raw_gtid.server_id)}:${raw_gtid.transaction_range}`;
+    return new ReplicatedGTID({
+      raw_gtid: stringGTID,
+      position
+    });
+  }
+
+  /**
+   * Special case for the zero GTID which means no transactions have been executed.
+   */
+  static ZERO = new ReplicatedGTID({ raw_gtid: '0:0', position: { filename: '', offset: 0 } });
+
+  constructor(protected options: ReplicatedGTIDSpecification) {}
+
+  /**
+   * Get the BinLog position of this replicated GTID event
+   */
+  get position() {
+    return this.options.position;
+  }
+
+  /**
+   * Get the raw Global Transaction ID. This of the format `server_id:transaction_ranges`
+   */
+  get raw() {
+    return this.options.raw_gtid;
+  }
+
+  get serverId() {
+    return this.options.raw_gtid.split(':')[0];
+  }
+
+  /**
+   * Transforms a GTID into a comparable string format, ensuring lexicographical
+   * order aligns with the GTID's relative age. This assumes that all GTIDs
+   * have the same server ID.
+   *
+   * @returns A comparable string in the format
+   *   `padded_end_transaction|raw_gtid|binlog_filename|binlog_position`
+   */
+  get comparable() {
+    const { raw, position } = this;
+    const [, transactionRanges] = this.raw.split(':');
+
+    let maxTransactionId = 0;
+
+    for (const range of transactionRanges.split(',')) {
+      const [start, end] = range.split('-');
+      maxTransactionId = Math.max(maxTransactionId, parseInt(start, 10), parseInt(end || start, 10));
+    }
+
+    const paddedTransactionId = maxTransactionId.toString().padStart(16, '0');
+    return [paddedTransactionId, raw, position.filename, position.offset].join('|');
+  }
+
+  toString() {
+    return this.comparable;
+  }
+
+  /**
+   * Calculates the distance in bytes from this GTID to the provided argument.
+   */
+  async distanceTo(connection: mysql.Connection, to: ReplicatedGTID): Promise<number | null> {
+    const [logFiles] = await mysql_utils.retriedQuery({
+      connection,
+      query: `SHOW BINARY LOGS;`
+    });
+
+    // Default to the first file for the start to handle the zero GTID case.
+    const startFileIndex = Math.max(
+      logFiles.findIndex((f) => f['Log_name'] == this.position.filename),
+      0
+    );
+    const startFileEntry = logFiles[startFileIndex];
+
+    if (!startFileEntry) {
+      return null;
+    }
+
+    /**
+     * Fall back to the next position for comparison if the replicated position is not present
+     */
+    const endPosition = to.position;
+
+    // Default to the past the last file to cater for the HEAD case
+    const testEndFileIndex = logFiles.findIndex((f) => f['Log_name'] == endPosition?.filename);
+    // If the endPosition is not defined and found. Fallback to the last file as the end
+    const endFileIndex = testEndFileIndex < 0 && !endPosition ? logFiles.length : logFiles.length - 1;
+
+    const endFileEntry = logFiles[endFileIndex];
+
+    if (!endFileEntry) {
+      return null;
+    }
+
+    return (
+      startFileEntry['File_size'] -
+      this.position.offset -
+      endFileEntry['File_size'] +
+      endPosition.offset +
+      logFiles.slice(startFileIndex + 1, endFileIndex).reduce((sum, file) => sum + file['File_size'], 0)
+    );
+  }
+}

package/src/common/check-source-configuration.ts

@@ -0,0 +1,59 @@
+import mysqlPromise from 'mysql2/promise';
+import * as mysql_utils from '../utils/mysql_utils.js';
+
+export async function checkSourceConfiguration(connection: mysqlPromise.Connection) {
+  const errors: string[] = [];
+  const [[result]] = await mysql_utils.retriedQuery({
+    connection,
+    query: `
+      SELECT
+        @@GLOBAL.gtid_mode AS gtid_mode,
+        @@GLOBAL.log_bin AS log_bin,
+        @@GLOBAL.server_id AS server_id,
+        @@GLOBAL.log_bin_basename AS binlog_file,
+        @@GLOBAL.log_bin_index AS binlog_index_file
+      `
+  });
+
+  if (result.gtid_mode != 'ON') {
+    errors.push(`GTID is not enabled, it is currently set to ${result.gtid_mode}. Please enable it.`);
+  }
+
+  if (result.log_bin != 1) {
+    errors.push('Binary logging is not enabled. Please enable it.');
+  }
+
+  if (result.server_id < 0) {
+    errors.push(
+      `Your Server ID setting is too low, it must be greater than 0. It is currently ${result.server_id}. Please correct your configuration.`
+    );
+  }
+
+  if (!result.binlog_file) {
+    errors.push('Binary log file is not set. Please check your settings.');
+  }
+
+  if (!result.binlog_index_file) {
+    errors.push('Binary log index file is not set. Please check your settings.');
+  }
+
+  const [[binLogFormatResult]] = await mysql_utils.retriedQuery({
+    connection,
+    query: `SHOW VARIABLES LIKE 'binlog_format';`
+  });
+
+  if (binLogFormatResult.Value !== 'ROW') {
+    errors.push('Binary log format must be set to "ROW". Please correct your configuration');
+  }
+
+  return errors;
+}
+
+export async function getMySQLVersion(connection: mysqlPromise.Connection): Promise<string> {
+  const [[versionResult]] = await mysql_utils.retriedQuery({
+    connection,
+    query: `SELECT VERSION() as version`
+  });
+
+  return versionResult.version as string;
+}

package/src/common/common-index.ts

@@ -0,0 +1,6 @@
+export * from './check-source-configuration.js';
+export * from './get-replication-columns.js';
+export * from './get-tables-from-pattern.js';
+export * from './mysql-to-sqlite.js';
+export * from './read-executed-gtid.js';
+export * from './ReplicatedGTID.js';

package/src/common/get-replication-columns.ts

@@ -0,0 +1,124 @@
+import { storage } from '@powersync/service-core';
+import mysqlPromise from 'mysql2/promise';
+import * as mysql_utils from '../utils/mysql_utils.js';
+
+export type GetReplicationColumnsOptions = {
+  connection: mysqlPromise.Connection;
+  schema: string;
+  table_name: string;
+};
+
+export type ReplicationIdentityColumnsResult = {
+  columns: storage.ColumnDescriptor[];
+  //   TODO maybe export an enum from the core package
+  identity: string;
+};
+
+export async function getReplicationIdentityColumns(
+  options: GetReplicationColumnsOptions
+): Promise<ReplicationIdentityColumnsResult> {
+  const { connection, schema, table_name } = options;
+  const [primaryKeyColumns] = await mysql_utils.retriedQuery({
+    connection: connection,
+    query: `
+      SELECT 
+        s.COLUMN_NAME AS name,
+        c.DATA_TYPE AS type
+      FROM 
+        INFORMATION_SCHEMA.STATISTICS s
+        JOIN 
+          INFORMATION_SCHEMA.COLUMNS c 
+            ON 
+              s.TABLE_SCHEMA = c.TABLE_SCHEMA
+              AND s.TABLE_NAME = c.TABLE_NAME
+              AND s.COLUMN_NAME = c.COLUMN_NAME
+      WHERE 
+        s.TABLE_SCHEMA = ?
+        AND s.TABLE_NAME = ?
+        AND s.INDEX_NAME = 'PRIMARY'
+      ORDER BY 
+        s.SEQ_IN_INDEX;
+      `,
+    params: [schema, table_name]
+  });
+
+  if (primaryKeyColumns.length) {
+    return {
+      columns: primaryKeyColumns.map((row) => ({
+        name: row.name,
+        type: row.type
+      })),
+      identity: 'default'
+    };
+  }
+
+  // TODO: test code with tables with unique keys, compound key etc.
+  // No primary key, find the first valid unique key
+  const [uniqueKeyColumns] = await mysql_utils.retriedQuery({
+    connection: connection,
+    query: `
+      SELECT 
+        s.INDEX_NAME,
+        s.COLUMN_NAME,
+        c.DATA_TYPE,
+        s.NON_UNIQUE,
+        s.NULLABLE
+      FROM 
+        INFORMATION_SCHEMA.STATISTICS s
+      JOIN 
+        INFORMATION_SCHEMA.COLUMNS c
+          ON 
+            s.TABLE_SCHEMA = c.TABLE_SCHEMA
+            AND s.TABLE_NAME = c.TABLE_NAME
+            AND s.COLUMN_NAME = c.COLUMN_NAME
+      WHERE 
+        s.TABLE_SCHEMA = ?
+        AND s.TABLE_NAME = ?
+        AND s.INDEX_NAME != 'PRIMARY'
+        AND s.NON_UNIQUE = 0
+      ORDER BY s.SEQ_IN_INDEX;
+      `,
+    params: [schema, table_name]
+  });
+
+  if (uniqueKeyColumns.length > 0) {
+    return {
+      columns: uniqueKeyColumns.map((col) => ({
+        name: col.COLUMN_NAME,
+        type: col.DATA_TYPE
+      })),
+      identity: 'index'
+    };
+  }
+
+  const [allColumns] = await mysql_utils.retriedQuery({
+    connection: connection,
+    query: `
+      SELECT 
+        s.COLUMN_NAME AS name,
+        c.DATA_TYPE as type
+      FROM 
+        INFORMATION_SCHEMA.COLUMNS s
+        JOIN 
+          INFORMATION_SCHEMA.COLUMNS c
+            ON 
+              s.TABLE_SCHEMA = c.TABLE_SCHEMA
+              AND s.TABLE_NAME = c.TABLE_NAME
+              AND s.COLUMN_NAME = c.COLUMN_NAME
+      WHERE 
+        s.TABLE_SCHEMA = ?
+        AND s.TABLE_NAME = ?
+      ORDER BY 
+        s.ORDINAL_POSITION;
+      `,
+    params: [schema, table_name]
+  });
+
+  return {
+    columns: allColumns.map((row) => ({
+      name: row.name,
+      type: row.type
+    })),
+    identity: 'full'
+  };
+}