@powersync/service-module-mysql 0.0.0-dev-20241015210820

package/src/common/get-tables-from-pattern.ts

@@ -0,0 +1,44 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import mysql from 'mysql2/promise';
+
+export type GetDebugTablesInfoOptions = {
+  connection: mysql.Connection;
+  tablePattern: sync_rules.TablePattern;
+};
+
+export async function getTablesFromPattern(options: GetDebugTablesInfoOptions): Promise<Set<string>> {
+  const { connection, tablePattern } = options;
+  const schema = tablePattern.schema;
+
+  if (tablePattern.isWildcard) {
+    const [results] = await connection.query<mysql.RowDataPacket[]>(
+      `SELECT
+            TABLE_NAME AS table_name
+           FROM 
+            INFORMATION_SCHEMA.TABLES
+           WHERE 
+            TABLE_SCHEMA = ?
+            AND TABLE_NAME LIKE ?`,
+      [schema, tablePattern.tablePattern]
+    );
+
+    return new Set(
+      results
+        .filter((result) => result.table_name.startsWith(tablePattern.tablePrefix))
+        .map((result) => result.table_name)
+    );
+  } else {
+    const [[match]] = await connection.query<mysql.RowDataPacket[]>(
+      `SELECT
+            TABLE_NAME AS table_name
+           FROM 
+            INFORMATION_SCHEMA.TABLES
+           WHERE 
+            TABLE_SCHEMA = ?
+            AND TABLE_NAME = ?`,
+      [tablePattern.schema, tablePattern.tablePattern]
+    );
+    // Only return the first result
+    return new Set([match.table_name]);
+  }
+}

package/src/common/mysql-to-sqlite.ts

@@ -0,0 +1,59 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import { ExpressionType } from '@powersync/service-sync-rules';
+
+export function toSQLiteRow(row: Record<string, any>): sync_rules.SqliteRow {
+  for (let key in row) {
+    if (row[key] instanceof Date) {
+      row[key] = row[key].toISOString();
+    }
+  }
+  return sync_rules.toSyncRulesRow(row);
+}
+
+export function toExpressionTypeFromMySQLType(mysqlType: string | undefined): ExpressionType {
+  if (!mysqlType) {
+    return ExpressionType.TEXT;
+  }
+
+  const upperCaseType = mysqlType.toUpperCase();
+  // Handle type with parameters like VARCHAR(255), DECIMAL(10,2), etc.
+  const baseType = upperCaseType.split('(')[0];
+
+  switch (baseType) {
+    case 'BIT':
+    case 'BOOL':
+    case 'BOOLEAN':
+    case 'TINYINT':
+    case 'SMALLINT':
+    case 'MEDIUMINT':
+    case 'INT':
+    case 'INTEGER':
+    case 'BIGINT':
+    case 'UNSIGNED BIGINT':
+      return ExpressionType.INTEGER;
+    case 'BINARY':
+    case 'VARBINARY':
+    case 'TINYBLOB':
+    case 'MEDIUMBLOB':
+    case 'LONGBLOB':
+    case 'BLOB':
+    case 'GEOMETRY':
+    case 'POINT':
+    case 'LINESTRING':
+    case 'POLYGON':
+    case 'MULTIPOINT':
+    case 'MULTILINESTRING':
+    case 'MULTIPOLYGON':
+    case 'GEOMETRYCOLLECTION':
+      return ExpressionType.BLOB;
+    case 'FLOAT':
+    case 'DOUBLE':
+    case 'REAL':
+      return ExpressionType.REAL;
+    case 'JSON':
+      return ExpressionType.TEXT;
+    default:
+      // In addition to the normal text types, includes: DECIMAL, NUMERIC, DATE, TIME, DATETIME, TIMESTAMP, YEAR, ENUM, SET
+      return ExpressionType.TEXT;
+  }
+}

package/src/common/read-executed-gtid.ts

@@ -0,0 +1,43 @@
+import mysqlPromise from 'mysql2/promise';
+import * as mysql_utils from '../utils/mysql_utils.js';
+import { gte } from 'semver';
+
+import { ReplicatedGTID } from './ReplicatedGTID.js';
+import { getMySQLVersion } from './check-source-configuration.js';
+import { logger } from '@powersync/lib-services-framework';
+
+/**
+ * Gets the current master HEAD GTID
+ */
+export async function readExecutedGtid(connection: mysqlPromise.Connection): Promise<ReplicatedGTID> {
+  const version = await getMySQLVersion(connection);
+  let binlogStatus: mysqlPromise.RowDataPacket;
+  if (gte(version, '8.4.0')) {
+    // Get the BinLog status
+    const [[binLogResult]] = await mysql_utils.retriedQuery({
+      connection,
+      query: `SHOW BINARY LOG STATUS`
+    });
+    binlogStatus = binLogResult;
+  } else {
+    // TODO Check if this works for version 5.7
+    // Get the BinLog status
+    const [[binLogResult]] = await mysql_utils.retriedQuery({
+      connection,
+      query: `SHOW MASTER STATUS`
+    });
+    binlogStatus = binLogResult;
+  }
+  const position = {
+    filename: binlogStatus.File,
+    offset: parseInt(binlogStatus.Position)
+  };
+
+  logger.info('Succesfully read executed GTID', { position });
+
+  return new ReplicatedGTID({
+    // The head always points to the next position to start replication from
+    position,
+    raw_gtid: binlogStatus.Executed_Gtid_Set
+  });
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+import { MySQLModule } from './module/MySQLModule.js';
+
+export const module = new MySQLModule();
+
+export default module;

package/src/module/MySQLModule.ts

@@ -0,0 +1,53 @@
+import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core';
+
+import { MySQLRouteAPIAdapter } from '../api/MySQLRouteAPIAdapter.js';
+import { BinLogReplicator } from '../replication/BinLogReplicator.js';
+import { MySQLErrorRateLimiter } from '../replication/MySQLErrorRateLimiter.js';
+import * as types from '../types/types.js';
+import { MySQLConnectionManagerFactory } from '../replication/MySQLConnectionManagerFactory.js';
+
+export class MySQLModule extends replication.ReplicationModule<types.MySQLConnectionConfig> {
+  constructor() {
+    super({
+      name: 'MySQL',
+      type: types.MYSQL_CONNECTION_TYPE,
+      configSchema: types.MySQLConnectionConfig
+    });
+  }
+
+  async initialize(context: system.ServiceContextContainer): Promise<void> {
+    await super.initialize(context);
+  }
+
+  protected createRouteAPIAdapter(): api.RouteAPI {
+    return new MySQLRouteAPIAdapter(this.resolveConfig(this.decodedConfig!));
+  }
+
+  protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
+    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
+    const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
+    const connectionFactory = new MySQLConnectionManagerFactory(normalisedConfig);
+
+    return new BinLogReplicator({
+      id: this.getDefaultId(normalisedConfig.database),
+      syncRuleProvider: syncRuleProvider,
+      storageEngine: context.storageEngine,
+      connectionFactory: connectionFactory,
+      rateLimiter: new MySQLErrorRateLimiter()
+    });
+  }
+
+  /**
+   * Combines base config with normalized connection settings
+   */
+  private resolveConfig(config: types.MySQLConnectionConfig): types.ResolvedConnectionConfig {
+    return {
+      ...config,
+      ...types.normalizeConnectionConfig(config)
+    };
+  }
+
+  async teardown(options: TearDownOptions): Promise<void> {
+    // No specific teardown required for MySQL
+  }
+}

package/src/replication/BinLogReplicationJob.ts

@@ -0,0 +1,97 @@
+import { container } from '@powersync/lib-services-framework';
+import { replication } from '@powersync/service-core';
+import { BinLogStream } from './BinLogStream.js';
+import { MySQLConnectionManagerFactory } from './MySQLConnectionManagerFactory.js';
+
+export interface BinLogReplicationJobOptions extends replication.AbstractReplicationJobOptions {
+  connectionFactory: MySQLConnectionManagerFactory;
+}
+
+export class BinLogReplicationJob extends replication.AbstractReplicationJob {
+  private connectionFactory: MySQLConnectionManagerFactory;
+
+  constructor(options: BinLogReplicationJobOptions) {
+    super(options);
+    this.connectionFactory = options.connectionFactory;
+  }
+
+  get slot_name() {
+    return this.options.storage.slot_name;
+  }
+
+  async keepAlive() {}
+
+  async replicate() {
+    try {
+      await this.replicateLoop();
+    } catch (e) {
+      // Fatal exception
+      container.reporter.captureException(e, {
+        metadata: {
+          replication_slot: this.slot_name
+        }
+      });
+      this.logger.error(`Replication failed on ${this.slot_name}`, e);
+
+      //   Slot removal type error logic goes here
+      //   if (e) {
+      //     // This stops replication on this slot, and creates a new slot
+      //     await this.options.storage.factory.slotRemoved(this.slot_name);
+      //   }
+    } finally {
+      this.abortController.abort();
+    }
+  }
+
+  async replicateLoop() {
+    while (!this.isStopped) {
+      await this.replicateOnce();
+
+      if (!this.isStopped) {
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+      }
+    }
+  }
+
+  async replicateOnce() {
+    // New connections on every iteration (every error with retry),
+    // otherwise we risk repeating errors related to the connection,
+    // such as caused by cached PG schemas.
+    const connectionManager = this.connectionFactory.create({
+      // Pool connections are only used intermittently.
+      idleTimeout: 30_000
+    });
+    try {
+      await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
+      if (this.isStopped) {
+        return;
+      }
+      const stream = new BinLogStream({
+        abortSignal: this.abortController.signal,
+        storage: this.options.storage,
+        connections: connectionManager
+      });
+      await stream.replicate();
+    } catch (e) {
+      this.logger.error(`Replication error`, e);
+      if (e.cause != null) {
+        this.logger.error(`cause`, e.cause);
+      }
+      // TODO not recoverable error
+      if (false) {
+        throw e;
+      } else {
+        // Report the error if relevant, before retrying
+        container.reporter.captureException(e, {
+          metadata: {
+            replication_slot: this.slot_name
+          }
+        });
+        // This sets the retry delay
+        this.rateLimiter?.reportError(e);
+      }
+    } finally {
+      await connectionManager.end();
+    }
+  }
+}

package/src/replication/BinLogReplicator.ts

@@ -0,0 +1,35 @@
+import { replication, storage } from '@powersync/service-core';
+import { BinLogReplicationJob } from './BinLogReplicationJob.js';
+import { MySQLConnectionManagerFactory } from './MySQLConnectionManagerFactory.js';
+
+export interface BinLogReplicatorOptions extends replication.AbstractReplicatorOptions {
+  connectionFactory: MySQLConnectionManagerFactory;
+}
+
+export class BinLogReplicator extends replication.AbstractReplicator<BinLogReplicationJob> {
+  private readonly connectionFactory: MySQLConnectionManagerFactory;
+
+  constructor(options: BinLogReplicatorOptions) {
+    super(options);
+    this.connectionFactory = options.connectionFactory;
+  }
+
+  createJob(options: replication.CreateJobOptions): BinLogReplicationJob {
+    return new BinLogReplicationJob({
+      id: this.createJobId(options.storage.group_id),
+      storage: options.storage,
+      lock: options.lock,
+      connectionFactory: this.connectionFactory,
+      rateLimiter: this.rateLimiter
+    });
+  }
+
+  async cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise<void> {
+    // The MySQL module does not create anything which requires cleanup on the MySQL server.
+  }
+
+  async stop(): Promise<void> {
+    await super.stop();
+    await this.connectionFactory.shutdown();
+  }
+}