@powersync/service-module-mysql 0.0.0-dev-20241015210820

package/src/replication/MySQLErrorRateLimiter.ts

@@ -0,0 +1,44 @@
+import { ErrorRateLimiter } from '@powersync/service-core';
+import { setTimeout } from 'timers/promises';
+
+export class MySQLErrorRateLimiter implements ErrorRateLimiter {
+  nextAllowed: number = Date.now();
+
+  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
+    const delay = Math.max(0, this.nextAllowed - Date.now());
+    // Minimum delay between connections, even without errors
+    this.setDelay(500);
+    await setTimeout(delay, undefined, { signal: options?.signal });
+  }
+
+  mayPing(): boolean {
+    return Date.now() >= this.nextAllowed;
+  }
+
+  reportError(e: any): void {
+    const message = (e.message as string) ?? '';
+    if (message.includes('password authentication failed')) {
+      // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+      this.setDelay(900_000);
+    } else if (message.includes('ENOTFOUND')) {
+      // DNS lookup issue - incorrect URI or deleted instance
+      this.setDelay(120_000);
+    } else if (message.includes('ECONNREFUSED')) {
+      // Could be fail2ban or similar
+      this.setDelay(120_000);
+    } else if (
+      message.includes('Unable to do postgres query on ended pool') ||
+      message.includes('Postgres unexpectedly closed connection')
+    ) {
+      // Connection timed out - ignore / immediately retry
+      // We don't explicitly set the delay to 0, since there could have been another error that
+      // we need to respect.
+    } else {
+      this.setDelay(30_000);
+    }
+  }
+
+  private setDelay(delay: number) {
+    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+  }
+}

package/src/replication/zongji/zongji-utils.ts

@@ -0,0 +1,32 @@
+import {
+  BinLogEvent,
+  BinLogGTIDLogEvent,
+  BinLogMutationEvent,
+  BinLogRotationEvent,
+  BinLogUpdateEvent,
+  BinLogXidEvent
+} from '@powersync/mysql-zongji';
+
+export function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent {
+  return event.getEventName() == 'gtidlog';
+}
+
+export function eventIsXid(event: BinLogEvent): event is BinLogXidEvent {
+  return event.getEventName() == 'xid';
+}
+
+export function eventIsRotation(event: BinLogEvent): event is BinLogRotationEvent {
+  return event.getEventName() == 'rotate';
+}
+
+export function eventIsWriteMutation(event: BinLogEvent): event is BinLogMutationEvent {
+  return event.getEventName() == 'writerows';
+}
+
+export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogMutationEvent {
+  return event.getEventName() == 'deleterows';
+}
+
+export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogUpdateEvent {
+  return event.getEventName() == 'updaterows';
+}

package/src/replication/zongji/zongji.d.ts

@@ -0,0 +1,98 @@
+declare module '@powersync/mysql-zongji' {
+  export type ZongjiOptions = {
+    host: string;
+    user: string;
+    password: string;
+  };
+
+  export type StartOptions = {
+    includeEvents?: string[];
+    excludeEvents?: string[];
+    /**
+     * BinLog position filename to start reading events from
+     */
+    filename?: string;
+    /**
+     * BinLog position offset to start reading events from in file specified
+     */
+    position?: number;
+  };
+
+  export type ColumnSchema = {
+    COLUMN_NAME: string;
+    COLLATION_NAME: string;
+    CHARACTER_SET_NAME: string;
+    COLUMN_COMMENT: string;
+    COLUMN_TYPE: string;
+  };
+
+  export type ColumnDefinition = {
+    name: string;
+    charset: string;
+    type: number;
+    metadata: Record<string, any>;
+  };
+
+  export type TableMapEntry = {
+    columnSchemas: Array<ColumnSchema>;
+    parentSchema: string;
+    tableName: string;
+    columns: Array<ColumnDefinition>;
+  };
+
+  export type BaseBinLogEvent = {
+    timestamp: number;
+    getEventName(): string;
+
+    /**
+     * Next position in BinLog file to read from after
+     * this event.
+     */
+    nextPosition: number;
+    /**
+     * Size of this event
+     */
+    size: number;
+    flags: number;
+    useChecksum: boolean;
+  };
+
+  export type BinLogRotationEvent = BaseBinLogEvent & {
+    binlogName: string;
+    position: number;
+  };
+
+  export type BinLogGTIDLogEvent = BaseBinLogEvent & {
+    serverId: Buffer;
+    transactionRange: number;
+  };
+
+  export type BinLogXidEvent = BaseBinLogEvent & {
+    xid: number;
+  };
+
+  export type BinLogMutationEvent = BaseBinLogEvent & {
+    tableId: number;
+    numberOfColumns: number;
+    tableMap: Record<string, TableMapEntry>;
+    rows: Array<Record<string, any>>;
+  };
+
+  export type BinLogUpdateEvent = Omit<BinLogMutationEvent, 'rows'> & {
+    rows: Array<{
+      before: Record<string, any>;
+      after: Record<string, any>;
+    }>;
+  };
+
+  export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent;
+
+  export default class ZongJi {
+    constructor(options: ZongjiOptions);
+
+    start(options: StartOptions): void;
+    stop(): void;
+
+    on(type: 'binlog' | string, callback: (event: BinLogEvent) => void);
+  }
+}

package/src/types/types.ts

@@ -0,0 +1,102 @@
+import * as service_types from '@powersync/service-types';
+import * as t from 'ts-codec';
+import * as urijs from 'uri-js';
+
+export const MYSQL_CONNECTION_TYPE = 'mysql' as const;
+
+export interface NormalizedMySQLConnectionConfig {
+  id: string;
+  tag: string;
+
+  hostname: string;
+  port: number;
+  database: string;
+
+  username: string;
+  password: string;
+
+  cacert?: string;
+  client_certificate?: string;
+  client_private_key?: string;
+}
+
+export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.and(
+  t.object({
+    type: t.literal(MYSQL_CONNECTION_TYPE),
+    uri: t.string.optional(),
+    hostname: t.string.optional(),
+    port: service_types.configFile.portCodec.optional(),
+    username: t.string.optional(),
+    password: t.string.optional(),
+    database: t.string.optional(),
+
+    cacert: t.string.optional(),
+    client_certificate: t.string.optional(),
+    client_private_key: t.string.optional()
+  })
+);
+
+/**
+ * Config input specified when starting services
+ */
+export type MySQLConnectionConfig = t.Decoded<typeof MySQLConnectionConfig>;
+
+/**
+ * Resolved version of {@link MySQLConnectionConfig}
+ */
+export type ResolvedConnectionConfig = MySQLConnectionConfig & NormalizedMySQLConnectionConfig;
+
+/**
+ * Validate and normalize connection options.
+ *
+ * Returns destructured options.
+ */
+export function normalizeConnectionConfig(options: MySQLConnectionConfig): NormalizedMySQLConnectionConfig {
+  let uri: urijs.URIComponents;
+  if (options.uri) {
+    uri = urijs.parse(options.uri);
+    if (uri.scheme != 'mysql') {
+      throw new Error(`Invalid URI - protocol must be mysql, got ${uri.scheme}`);
+    }
+  } else {
+    uri = urijs.parse('mysql:///');
+  }
+
+  const hostname = options.hostname ?? uri.host ?? '';
+  const port = Number(options.port ?? uri.port ?? 3306);
+
+  const database = options.database ?? uri.path?.substring(1) ?? '';
+
+  const [uri_username, uri_password] = (uri.userinfo ?? '').split(':');
+
+  const username = options.username ?? uri_username ?? '';
+  const password = options.password ?? uri_password ?? '';
+
+  if (hostname == '') {
+    throw new Error(`hostname required`);
+  }
+
+  if (username == '') {
+    throw new Error(`username required`);
+  }
+
+  if (password == '') {
+    throw new Error(`password required`);
+  }
+
+  if (database == '') {
+    throw new Error(`database required`);
+  }
+
+  return {
+    id: options.id ?? 'default',
+    tag: options.tag ?? 'default',
+
+    hostname,
+    port,
+    database,
+
+    username,
+    password
+  };
+}

package/src/utils/mysql_utils.ts

@@ -0,0 +1,47 @@
+import { logger } from '@powersync/lib-services-framework';
+import mysql from 'mysql2';
+import mysqlPromise from 'mysql2/promise';
+import * as types from '../types/types.js';
+
+export type RetriedQueryOptions = {
+  connection: mysqlPromise.Connection;
+  query: string;
+  params?: any[];
+  retries?: number;
+};
+
+/**
+ * Retry a simple query - up to 2 attempts total.
+ */
+export async function retriedQuery(options: RetriedQueryOptions) {
+  const { connection, query, params = [], retries = 2 } = options;
+  for (let tries = retries; ; tries--) {
+    try {
+      logger.debug(`Executing query: ${query}`);
+      return connection.query<mysqlPromise.RowDataPacket[]>(query, params);
+    } catch (e) {
+      if (tries == 1) {
+        throw e;
+      }
+      logger.warn('Query error, retrying', e);
+    }
+  }
+}
+
+export function createPool(config: types.NormalizedMySQLConnectionConfig, options?: mysql.PoolOptions): mysql.Pool {
+  const sslOptions = {
+    ca: config.cacert,
+    key: config.client_private_key,
+    cert: config.client_certificate
+  };
+  const hasSSLOptions = Object.values(sslOptions).some((v) => !!v);
+  // TODO confirm if default options are fine for Powersync use case
+  return mysql.createPool({
+    host: config.hostname,
+    user: config.username,
+    password: config.password,
+    database: config.database,
+    ssl: hasSSLOptions ? sslOptions : undefined,
+    ...(options || {})
+  });
+}

package/test/src/binlog_stream.test.ts

@@ -0,0 +1,288 @@
+import { putOp, removeOp } from '@core-tests/stream_utils.js';
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { BucketStorageFactory, Metrics } from '@powersync/service-core';
+import * as crypto from 'crypto';
+import { describe, expect, test } from 'vitest';
+import { binlogStreamTest } from './binlog_stream_utils.js';
+import { logger } from '@powersync/lib-services-framework';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+
+describe(
+  ' Binlog stream - mongodb',
+  function () {
+    defineBinlogStreamTests(MONGO_STORAGE_FACTORY);
+  },
+  { timeout: 20_000 }
+);
+
+function defineBinlogStreamTests(factory: StorageFactory) {
+  test(
+    'Replicate basic values',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description, num FROM "test_data"`);
+
+      await connectionManager.query(
+        `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description TEXT, num BIGINT)`
+      );
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+      await connectionManager.query(`INSERT INTO test_data(description, num) VALUES('test1', 1152921504606846976)`);
+      const [[result]] = await connectionManager.query(
+        `SELECT id AS test_id FROM test_data WHERE description = 'test1' AND num = 1152921504606846976`
+      );
+      const testId = result.test_id;
+      logger.info('Finished Inserting data with id:' + testId);
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  // test(
+  //   'replicating case sensitive table',
+  //   binlogStreamTest(factory, async (context) => {
+  //     const { connectionManager } = context;
+  //     await context.updateSyncRules(`
+  //     bucket_definitions:
+  //       global:
+  //         data:
+  //           - SELECT id, description FROM "test_DATA"
+  //     `);
+
+  //     await connectionManager.query(
+  //       `CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
+  //     );
+
+  //     await context.replicateSnapshot();
+
+  //     const startRowCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+  //     const startTxCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+  //     context.startStreaming();
+
+  //     await connectionManager.query(`INSERT INTO test_DATA(description) VALUES('test1')`);
+  //     const [[result]] = await connectionManager.query(
+  //       `SELECT id AS test_id FROM test_data WHERE description = 'test1'`
+  //     );
+  //     const testId = result.test_id;
+
+  //     const data = await context.getBucketData('global[]');
+
+  //     expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]);
+  //     const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+  //     const endTxCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+  //     expect(endRowCount - startRowCount).toEqual(1);
+  //     expect(endTxCount - startTxCount).toEqual(1);
+  //   })
+  // );
+
+  // //   TODO: Not supported yet
+  // //   test(
+  // //     'replicating TRUNCATE',
+  // //     binlogStreamTest(factory, async (context) => {
+  // //       const { connectionManager } = context;
+  // //       const syncRuleContent = `
+  // // bucket_definitions:
+  // //   global:
+  // //     data:
+  // //       - SELECT id, description FROM "test_data"
+  // //   by_test_data:
+  // //     parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
+  // //     data: []
+  // // `;
+  // //       await context.updateSyncRules(syncRuleContent);
+  // //       await connectionManager.query(`DROP TABLE IF EXISTS test_data`);
+  // //       await connectionManager.query(
+  // //         `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`
+  // //       );
+  // //
+  // //       await context.replicateSnapshot();
+  // //       context.startStreaming();
+  // //
+  // //       const [{ test_id }] = pgwireRows(
+  // //         await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+  // //       );
+  // //       await connectionManager.query(`TRUNCATE test_data`);
+  // //
+  // //       const data = await context.getBucketData('global[]');
+  // //
+  // //       expect(data).toMatchObject([
+  // //         putOp('test_data', { id: test_id, description: 'test1' }),
+  // //         removeOp('test_data', test_id)
+  // //       ]);
+  // //     })
+  // //   );
+
+  // test(
+  //   'replicating changing primary key',
+  //   binlogStreamTest(factory, async (context) => {
+  //     const { connectionManager } = context;
+  //     await context.updateSyncRules(BASIC_SYNC_RULES);
+
+  //     await connectionManager.query(
+  //       `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
+  //     );
+
+  //     await context.replicateSnapshot();
+  //     context.startStreaming();
+
+  //     await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1')`);
+  //     const [[result1]] = await connectionManager.query(
+  //       `SELECT id AS test_id FROM test_data WHERE description = 'test1'`
+  //     );
+  //     const testId1 = result1.test_id;
+
+  //     await connectionManager.query(`UPDATE test_data SET id = UUID(), description = 'test2a' WHERE id = '${testId1}'`);
+  //     const [[result2]] = await connectionManager.query(
+  //       `SELECT id AS test_id FROM test_data WHERE description = 'test2a'`
+  //     );
+  //     const testId2 = result2.test_id;
+
+  //     // This update may fail replicating with:
+  //     // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
+  //     await connectionManager.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${testId2}'`);
+
+  //     // Re-use old id again
+  //     await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}', 'test1b')`);
+  //     await connectionManager.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${testId1}'`);
+
+  //     const data = await context.getBucketData('global[]');
+  //     expect(data).toMatchObject([
+  //       // Initial insert
+  //       putOp('test_data', { id: testId1, description: 'test1' }),
+  //       // Update id, then description
+  //       removeOp('test_data', testId1),
+  //       putOp('test_data', { id: testId2, description: 'test2a' }),
+  //       putOp('test_data', { id: testId2, description: 'test2b' }),
+  //       // Re-use old id
+  //       putOp('test_data', { id: testId1, description: 'test1b' }),
+  //       putOp('test_data', { id: testId1, description: 'test1c' })
+  //     ]);
+  //   })
+  // );
+
+  // test(
+  //   'initial sync',
+  //   binlogStreamTest(factory, async (context) => {
+  //     const { connectionManager } = context;
+  //     await context.updateSyncRules(BASIC_SYNC_RULES);
+
+  //     await connectionManager.query(
+  //       `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
+  //     );
+
+  //     await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1')`);
+  //     const [[result]] = await connectionManager.query(
+  //       `SELECT id AS test_id FROM test_data WHERE description = 'test1'`
+  //     );
+  //     const testId = result.test_id;
+
+  //     await context.replicateSnapshot();
+  //     context.startStreaming();
+
+  //     const data = await context.getBucketData('global[]');
+  //     expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]);
+  //   })
+  // );
+
+  // // test(
+  // //   'record too large',
+  // //   binlogStreamTest(factory, async (context) => {
+  // //     await context.updateSyncRules(`bucket_definitions:
+  // //     global:
+  // //       data:
+  // //         - SELECT id, description, other FROM "test_data"`);
+  // //     const { connectionManager } = context;
+  // //
+  // //     await connectionManager.query(`CREATE TABLE test_data(id text primary key, description text, other text)`);
+  // //
+  // //     await context.replicateSnapshot();
+  // //
+  // //     // 4MB
+  // //     const largeDescription = crypto.randomBytes(2_000_000).toString('hex');
+  // //     // 18MB
+  // //     const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex');
+  // //
+  // //     await connectionManager.query({
+  // //       statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`,
+  // //       params: [{ type: 'varchar', value: tooLargeDescription }]
+  // //     });
+  // //     await connectionManager.query({
+  // //       statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`,
+  // //       params: [{ type: 'varchar', value: largeDescription }]
+  // //     });
+  // //
+  // //     context.startStreaming();
+  // //
+  // //     const data = await context.getBucketData('global[]');
+  // //     expect(data.length).toEqual(1);
+  // //     const row = JSON.parse(data[0].data as string);
+  // //     delete row.description;
+  // //     expect(row).toEqual({ id: 't1', other: 'foo' });
+  // //     delete data[0].data;
+  // //     expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' });
+  // //   })
+  // // );
+
+  // test(
+  //   'table not in sync rules',
+  //   binlogStreamTest(factory, async (context) => {
+  //     const { connectionManager } = context;
+  //     await context.updateSyncRules(BASIC_SYNC_RULES);
+
+  //     await connectionManager.query(
+  //       `CREATE TABLE test_donotsync (id CHAR(36) PRIMARY KEY DEFAULT (UUID()), description text)`
+  //     );
+
+  //     await context.replicateSnapshot();
+
+  //     const startRowCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+  //     const startTxCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+  //     context.startStreaming();
+
+  //     await connectionManager.query(`INSERT INTO test_donotsync(description) VALUES('test1')`);
+  //     const data = await context.getBucketData('global[]');
+
+  //     expect(data).toMatchObject([]);
+  //     const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+  //     const endTxCount =
+  //       (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+  //     // There was a transaction, but we should not replicate any actual data
+  //     expect(endRowCount - startRowCount).toEqual(0);
+  //     expect(endTxCount - startTxCount).toEqual(1);
+  //   })
+  // );
+}