@powersync/service-module-mysql 0.0.0-dev-20241015210820

package/test/src/binlog_stream_utils.ts

@@ -0,0 +1,152 @@
+import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core';
+import { TEST_CONNECTION_OPTIONS, clearAndRecreateTestDb } from './util.js';
+import { fromAsync } from '@core-tests/stream_utils.js';
+import { BinLogStream, BinLogStreamOptions } from '@module/replication/BinLogStream.js';
+import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import mysqlPromise from 'mysql2/promise';
+import { readExecutedGtid } from '@module/common/read-executed-gtid.js';
+import { logger } from '@powersync/lib-services-framework';
+
+/**
+ * Tests operating on the binlog stream need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps a test in a function that configures all the context, and tears it down afterward.
+ */
+export function binlogStreamTest(
+  factory: () => Promise<BucketStorageFactory>,
+  test: (context: BinlogStreamTestContext) => Promise<void>
+): () => Promise<void> {
+  return async () => {
+    const f = await factory();
+    const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
+
+    const connection = await connectionManager.getConnection();
+    await clearAndRecreateTestDb(connection);
+    connection.release();
+    const context = new BinlogStreamTestContext(f, connectionManager);
+    try {
+      await test(context);
+    } finally {
+      await context.dispose();
+    }
+  };
+}
+
+export class BinlogStreamTestContext {
+  private _binlogStream?: BinLogStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+  private replicationDone = false;
+
+  constructor(
+    public factory: BucketStorageFactory,
+    public connectionManager: MySQLConnectionManager
+  ) {}
+
+  async dispose() {
+    this.abortController.abort();
+    await this.streamPromise;
+    await this.connectionManager.end();
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules({ content: content });
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  get binlogStream() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._binlogStream) {
+      return this._binlogStream;
+    }
+    const options: BinLogStreamOptions = {
+      storage: this.storage,
+      connections: this.connectionManager,
+      abortSignal: this.abortController.signal
+    };
+    this._binlogStream = new BinLogStream(options);
+    return this._binlogStream!;
+  }
+
+  async replicateSnapshot() {
+    await this.binlogStream.initReplication();
+    this.replicationDone = true;
+  }
+
+  startStreaming() {
+    if (!this.replicationDone) {
+      throw new Error('Call replicateSnapshot() before startStreaming()');
+    }
+    this.streamPromise = this.binlogStream.streamChanges();
+  }
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    const connection = await this.connectionManager.getConnection();
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(connection, this.factory, { timeout: options?.timeout ?? 60_000 }),
+      this.streamPromise
+    ]);
+    connection.release();
+    if (typeof checkpoint == undefined) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint as string;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>(Object.entries(buckets));
+    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
+    start ??= '0';
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>([[bucket, start]]);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.batch.data ?? [];
+  }
+}
+
+export async function getClientCheckpoint(
+  connection: mysqlPromise.Connection,
+  bucketStorage: BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+  const gtid = await readExecutedGtid(connection);
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+  let lastCp: ActiveCheckpoint | null = null;
+
+  logger.info('Expected Checkpoint: ' + gtid.comparable);
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    lastCp = cp;
+    //logger.info('Last Checkpoint: ' + lastCp.lsn);
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn && cp.lsn >= gtid.comparable) {
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error(`Timeout while waiting for checkpoint ${gtid.comparable}. Last checkpoint: ${lastCp?.lsn}`);
+}

package/test/src/env.ts

@@ -0,0 +1,7 @@
+import { utils } from '@powersync/lib-services-framework';
+
+export const env = utils.collectEnvironmentVariables({
+  MYSQL_TEST_URI: utils.type.string.default('mysql://myuser:mypassword@localhost:3306/mydatabase'),
+  CI: utils.type.boolean.default('false'),
+  SLOW_TESTS: utils.type.boolean.default('false')
+});

package/test/src/setup.ts

@@ -0,0 +1,7 @@
+import { container } from '@powersync/lib-services-framework';
+import { beforeAll } from 'vitest';
+
+beforeAll(() => {
+  // Executes for every test file
+  container.registerDefaults();
+});

package/test/src/util.ts

@@ -0,0 +1,62 @@
+import * as types from '@module/types/types.js';
+import { BucketStorageFactory, Metrics, MongoBucketStorage } from '@powersync/service-core';
+import { env } from './env.js';
+import mysqlPromise from 'mysql2/promise';
+import { connectMongo } from '@core-tests/util.js';
+import { getMySQLVersion } from '@module/common/check-source-configuration.js';
+import { gte } from 'semver';
+import { RowDataPacket } from 'mysql2';
+
+// The metrics need to be initialized before they can be used
+await Metrics.initialise({
+  disable_telemetry_sharing: true,
+  powersync_instance_id: 'test',
+  internal_metrics_endpoint: 'unused.for.tests.com'
+});
+Metrics.getInstance().resetCounters();
+
+export const TEST_URI = env.MYSQL_TEST_URI;
+
+export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
+  type: 'mysql',
+  uri: TEST_URI
+});
+
+export type StorageFactory = () => Promise<BucketStorageFactory>;
+
+export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
+  const db = await connectMongo();
+
+  // None of the tests insert data into this collection, so it was never created
+  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+    await db.db.createCollection('bucket_parameters');
+  }
+
+  await db.clear();
+
+  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
+};
+
+export async function clearAndRecreateTestDb(connection: mysqlPromise.Connection) {
+  const version = await getMySQLVersion(connection);
+  if (gte(version, '8.4.0')) {
+    await connection.query('RESET BINARY LOGS AND GTIDS');
+  } else {
+    await connection.query('RESET MASTER');
+  }
+
+  // await connection.query(`DROP DATABASE IF EXISTS ${TEST_CONNECTION_OPTIONS.database}`);
+  //
+  // await connection.query(`CREATE DATABASE IF NOT EXISTS ${TEST_CONNECTION_OPTIONS.database}`);
+
+  const [result] = await connection.query<RowDataPacket[]>(
+    `SELECT TABLE_NAME FROM information_schema.tables
+     WHERE TABLE_SCHEMA = '${TEST_CONNECTION_OPTIONS.database}'`
+  );
+  for (let row of result) {
+    const name = row.TABLE_NAME;
+    if (name.startsWith('test_')) {
+      await connection.query(`DROP TABLE ${name}`);
+    }
+  }
+}

package/test/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "baseUrl": "./",
+    "noEmit": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "paths": {
+      "@/*": ["../../../packages/service-core/src/*"],
+      "@module/*": ["../src/*"],
+      "@core-tests/*": ["../../../packages/service-core/test/src/*"]
+    }
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    },
+    {
+      "path": "../../../packages/service-core/test"
+    },
+    {
+      "path": "../../../packages/service-core/"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "typeRoots": ["./node_modules/@types", "./src/replication/zongji.d.ts"]
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/types"
+    },
+    {
+      "path": "../../packages/sync-rules"
+    },
+    {
+      "path": "../../packages/service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
+    }
+  ]
+}