@powersync/service-module-mysql 0.0.0-dev-20241101083236 → 0.0.0-dev-20241111122558

package/src/replication/BinLogStream.ts

@@ -9,9 +9,9 @@ import { BinLogEvent, StartOptions, TableMapEntry } from '@powersync/mysql-zongj
 import * as common from '../common/common-index.js';
 import * as zongji_utils from './zongji/zongji-utils.js';
 import { MySQLConnectionManager } from './MySQLConnectionManager.js';
-import { isBinlogStillAvailable, ReplicatedGTID } from '../common/common-index.js';
+import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js';
 import mysqlPromise from 'mysql2/promise';
-import { MySQLTypesMap } from '../utils/mysql_utils.js';
+import { createRandomServerId } from '../utils/mysql-utils.js';
 
 export interface BinLogStreamOptions {
   connections: MySQLConnectionManager;
@@ -221,7 +221,13 @@ AND table_type = 'BASE TABLE';`,
         // Check if the binlog is still available. If it isn't we need to snapshot again.
         const connection = await this.connections.getConnection();
         try {
-          return await isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
+          const isAvailable = await isBinlogStillAvailable(connection, lastKnowGTID.position.filename);
+          if (!isAvailable) {
+            logger.info(
+              `Binlog file ${lastKnowGTID.position.filename} is no longer available, starting initial replication again.`
+            );
+          }
+          return isAvailable;
         } finally {
           connection.release();
         }
@@ -245,7 +251,7 @@ AND table_type = 'BASE TABLE';`,
     const connection = await this.connections.getStreamingConnection();
     const promiseConnection = (connection as mysql.Connection).promise();
     const headGTID = await common.readExecutedGtid(promiseConnection);
-    logger.info(`Using snapshot checkpoint GTID:: '${headGTID}'`);
+    logger.info(`Using snapshot checkpoint GTID: '${headGTID}'`);
     try {
       logger.info(`Starting initial replication`);
       await promiseConnection.query<mysqlPromise.RowDataPacket[]>(
@@ -285,7 +291,7 @@ AND table_type = 'BASE TABLE';`,
     logger.info(`Replicating ${table.qualifiedName}`);
     // TODO count rows and log progress at certain batch sizes
 
-    const columns = new Map<string, ColumnDescriptor>();
+    let columns: Map<string, ColumnDescriptor>;
     return new Promise<void>((resolve, reject) => {
       // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
       connection
@@ -295,10 +301,7 @@ AND table_type = 'BASE TABLE';`,
         })
         .on('fields', (fields: FieldPacket[]) => {
           // Map the columns and their types
-          fields.forEach((field) => {
-            const columnType = MySQLTypesMap[field.type as number];
-            columns.set(field.name, { name: field.name, type: columnType, typeId: field.type });
-          });
+          columns = toColumnDescriptors(fields);
         })
         .on('result', async (row) => {
           connection.pause();
@@ -363,10 +366,14 @@ AND table_type = 'BASE TABLE';`,
   async streamChanges() {
     // Auto-activate as soon as initial replication is done
     await this.storage.autoActivate();
+    const serverId = createRandomServerId(this.storage.group_id);
+    logger.info(`Starting replication. Created replica client with serverId:${serverId}`);
 
     const connection = await this.connections.getConnection();
     const { checkpoint_lsn } = await this.storage.getStatus();
-    logger.info(`Last known LSN from storage: ${checkpoint_lsn}`);
+    if (checkpoint_lsn) {
+      logger.info(`Existing checkpoint found: ${checkpoint_lsn}`);
+    }
 
     const fromGTID = checkpoint_lsn
       ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn)
@@ -447,7 +454,7 @@ AND table_type = 'BASE TABLE';`,
 
           zongji.on('binlog', (evt: BinLogEvent) => {
             if (!this.stopped) {
-              logger.info(`Pushing Binlog event ${evt.getEventName()}`);
+              logger.info(`Received Binlog event:${evt.getEventName()}`);
               queue.push(evt);
             } else {
               logger.info(`Replication is busy stopping, ignoring event ${evt.getEventName()}`);
@@ -458,16 +465,18 @@ AND table_type = 'BASE TABLE';`,
             // Powersync is shutting down, don't start replicating
             return;
           }
+
+          logger.info(`Reading binlog from: ${binLogPositionState.filename}:${binLogPositionState.offset}`);
+
           // Only listen for changes to tables in the sync rules
           const includedTables = [...this.tableCache.values()].map((table) => table.table);
-          logger.info(`Starting replication from ${binLogPositionState.filename}:${binLogPositionState.offset}`);
           zongji.start({
             includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
             excludeEvents: [],
             includeSchema: { [this.defaultSchema]: includedTables },
             filename: binLogPositionState.filename,
             position: binLogPositionState.offset,
-            serverId: this.storage.group_id
+            serverId: serverId
           } satisfies StartOptions);
 
           // Forever young
@@ -516,10 +525,7 @@ AND table_type = 'BASE TABLE';`,
       tableEntry: TableMapEntry;
     }
   ): Promise<storage.FlushedResult | null> {
-    const columns = new Map<string, ColumnDescriptor>();
-    msg.tableEntry.columns.forEach((column) => {
-      columns.set(column.name, { name: column.name, typeId: column.type });
-    });
+    const columns = toColumnDescriptors(msg.tableEntry);
 
     for (const [index, row] of msg.data.entries()) {
       await this.writeChange(batch, {
@@ -560,8 +566,10 @@ AND table_type = 'BASE TABLE';`,
         Metrics.getInstance().rows_replicated_total.add(1);
         // "before" may be null if the replica id columns are unchanged
         // It's fine to treat that the same as an insert.
-        const beforeUpdated = payload.previous_data ? common.toSQLiteRow(payload.previous_data) : undefined;
-        const after = common.toSQLiteRow(payload.data);
+        const beforeUpdated = payload.previous_data
+          ? common.toSQLiteRow(payload.previous_data, payload.columns)
+          : undefined;
+        const after = common.toSQLiteRow(payload.data, payload.columns);
 
         return await batch.save({
           tag: storage.SaveOperationTag.UPDATE,
@@ -570,13 +578,13 @@ AND table_type = 'BASE TABLE';`,
           beforeReplicaId: beforeUpdated
             ? getUuidReplicaIdentityBson(beforeUpdated, payload.sourceTable.replicaIdColumns)
             : undefined,
-          after: common.toSQLiteRow(payload.data),
+          after: common.toSQLiteRow(payload.data, payload.columns),
           afterReplicaId: getUuidReplicaIdentityBson(after, payload.sourceTable.replicaIdColumns)
         });
 
       case storage.SaveOperationTag.DELETE:
         Metrics.getInstance().rows_replicated_total.add(1);
-        const beforeDeleted = common.toSQLiteRow(payload.data);
+        const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns);
 
         return await batch.save({
           tag: storage.SaveOperationTag.DELETE,

package/src/replication/MySQLConnectionManager.ts

@@ -1,7 +1,7 @@
 import { NormalizedMySQLConnectionConfig } from '../types/types.js';
 import mysqlPromise from 'mysql2/promise';
-import mysql, { RowDataPacket } from 'mysql2';
-import * as mysql_utils from '../utils/mysql_utils.js';
+import mysql, { FieldPacket, RowDataPacket } from 'mysql2';
+import * as mysql_utils from '../utils/mysql-utils.js';
 import ZongJi from '@powersync/mysql-zongji';
 import { logger } from '@powersync/lib-services-framework';
 
@@ -61,7 +61,7 @@ export class MySQLConnectionManager {
    *  @param query
    *  @param params
    */
-  async query(query: string, params?: any[]) {
+  async query(query: string, params?: any[]): Promise<[RowDataPacket[], FieldPacket[]]> {
     return this.promisePool.query<RowDataPacket[]>(query, params);
   }
 

package/src/utils/{mysql_utils.ts → mysql-utils.ts}

@@ -2,11 +2,7 @@ import { logger } from '@powersync/lib-services-framework';
 import mysql from 'mysql2';
 import mysqlPromise from 'mysql2/promise';
 import * as types from '../types/types.js';
-
-export const MySQLTypesMap: { [key: number]: string } = {};
-for (const [name, code] of Object.entries(mysql.Types)) {
-  MySQLTypesMap[code as number] = name;
-}
+import { coerce, gte } from 'semver';
 
 export type RetriedQueryOptions = {
   connection: mysqlPromise.Connection;
@@ -47,7 +43,42 @@ export function createPool(config: types.NormalizedMySQLConnectionConfig, option
     database: config.database,
     ssl: hasSSLOptions ? sslOptions : undefined,
     supportBigNumbers: true,
+    decimalNumbers: true,
     timezone: 'Z', // Ensure no auto timezone manipulation of the dates occur
+    jsonStrings: true, // Return JSON columns as strings
     ...(options || {})
   });
 }
+
+/**
+ *  Return a random server id for a given sync rule id.
+ *  Expected format is: <syncRuleId>00<random number>
+ *  The max value for server id in MySQL is 2^32 - 1.
+ *  We use the GTID format to keep track of our position in the binlog, no state is kept by the MySQL server, therefore
+ *  it is ok to use a randomised server id every time.
+ *  @param syncRuleId
+ */
+export function createRandomServerId(syncRuleId: number): number {
+  return Number.parseInt(`${syncRuleId}00${Math.floor(Math.random() * 10000)}`);
+}
+
+export async function getMySQLVersion(connection: mysqlPromise.Connection): Promise<string> {
+  const [[versionResult]] = await retriedQuery({
+    connection,
+    query: `SELECT VERSION() as version`
+  });
+
+  return versionResult.version as string;
+}
+
+/**
+ *  Check if the current MySQL version is newer or equal to the target version.
+ *  @param version
+ *  @param minimumVersion
+ */
+export function isVersionAtLeast(version: string, minimumVersion: string): boolean {
+  const coercedVersion = coerce(version);
+  const coercedMinimumVersion = coerce(minimumVersion);
+
+  return gte(coercedVersion!, coercedMinimumVersion!, { loose: true });
+}

package/test/src/BinLogStream.test.ts

@@ -0,0 +1,306 @@
+import { putOp, removeOp } from '@core-tests/stream_utils.js';
+import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js';
+import { BucketStorageFactory, Metrics } from '@powersync/service-core';
+import { describe, expect, test } from 'vitest';
+import { binlogStreamTest } from './BinlogStreamUtils.js';
+import { v4 as uuid } from 'uuid';
+
+type StorageFactory = () => Promise<BucketStorageFactory>;
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+
+describe(
+  ' Binlog stream - mongodb',
+  function () {
+    defineBinlogStreamTests(MONGO_STORAGE_FACTORY);
+  },
+  { timeout: 20_000 }
+);
+
+function defineBinlogStreamTests(factory: StorageFactory) {
+  test(
+    'Replicate basic values',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(`
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description, num FROM "test_data"`);
+
+      await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, num BIGINT)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+      const testId = uuid();
+      await connectionManager.query(
+        `INSERT INTO test_data(id, description, num) VALUES('${testId}', 'test1', 1152921504606846976)`
+      );
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'replicating case sensitive table',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, description FROM "test_DATA"
+      `);
+
+      await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const testId = uuid();
+      await connectionManager.query(`INSERT INTO test_DATA(id, description) VALUES('${testId}','test1')`);
+
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  //   TODO: Not supported yet
+  //   test(
+  //     'replicating TRUNCATE',
+  //     binlogStreamTest(factory, async (context) => {
+  //       const { connectionManager } = context;
+  //       const syncRuleContent = `
+  // bucket_definitions:
+  //   global:
+  //     data:
+  //       - SELECT id, description FROM "test_data"
+  //   by_test_data:
+  //     parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id
+  //     data: []
+  // `;
+  //       await context.updateSyncRules(syncRuleContent);
+  //       await connectionManager.query(`DROP TABLE IF EXISTS test_data`);
+  //       await connectionManager.query(
+  //         `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`
+  //       );
+  //
+  //       await context.replicateSnapshot();
+  //       context.startStreaming();
+  //
+  //       const [{ test_id }] = pgwireRows(
+  //         await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`)
+  //       );
+  //       await connectionManager.query(`TRUNCATE test_data`);
+  //
+  //       const data = await context.getBucketData('global[]');
+  //
+  //       expect(data).toMatchObject([
+  //         putOp('test_data', { id: test_id, description: 'test1' }),
+  //         removeOp('test_data', test_id)
+  //       ]);
+  //     })
+  //   );
+
+  test(
+    'replicating changing primary key',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`);
+
+      await context.replicateSnapshot();
+      context.startStreaming();
+
+      const testId1 = uuid();
+      await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}','test1')`);
+
+      const testId2 = uuid();
+      await connectionManager.query(
+        `UPDATE test_data SET id = '${testId2}', description = 'test2a' WHERE id = '${testId1}'`
+      );
+
+      // This update may fail replicating with:
+      // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"}
+      await connectionManager.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${testId2}'`);
+
+      // Re-use old id again
+      await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}', 'test1b')`);
+      await connectionManager.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${testId1}'`);
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([
+        // Initial insert
+        putOp('test_data', { id: testId1, description: 'test1' }),
+        // Update id, then description
+        removeOp('test_data', testId1),
+        putOp('test_data', { id: testId2, description: 'test2a' }),
+        putOp('test_data', { id: testId2, description: 'test2b' }),
+        // Re-use old id
+        putOp('test_data', { id: testId1, description: 'test1b' }),
+        putOp('test_data', { id: testId1, description: 'test1c' })
+      ]);
+    })
+  );
+
+  test(
+    'initial sync',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`);
+
+      const testId = uuid();
+      await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`);
+
+      await context.replicateSnapshot();
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]);
+    })
+  );
+
+  test(
+    'snapshot with date values',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT * FROM "test_data"
+      `);
+
+      await connectionManager.query(
+        `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, date DATE, datetime DATETIME, timestamp TIMESTAMP)`
+      );
+
+      const testId = uuid();
+      await connectionManager.query(`
+        INSERT INTO test_data(id, description, date, datetime, timestamp) VALUES('${testId}','testDates', '2023-03-06', '2023-03-06 15:47', '2023-03-06 15:47')
+      `);
+
+      await context.replicateSnapshot();
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([
+        putOp('test_data', {
+          id: testId,
+          description: 'testDates',
+          date: `2023-03-06`,
+          datetime: '2023-03-06T15:47:00.000Z',
+          timestamp: '2023-03-06T15:47:00.000Z'
+        })
+      ]);
+    })
+  );
+
+  test(
+    'replication with date values',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT * FROM "test_data"
+      `);
+
+      await connectionManager.query(
+        `CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, date DATE, datetime DATETIME, timestamp TIMESTAMP)`
+      );
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      const testId = uuid();
+      await connectionManager.query(`
+        INSERT INTO test_data(id, description, date, datetime, timestamp) VALUES('${testId}','testDates', '2023-03-06', '2023-03-06 15:47', '2023-03-06 15:47')
+      `);
+
+      const data = await context.getBucketData('global[]');
+      expect(data).toMatchObject([
+        putOp('test_data', {
+          id: testId,
+          description: 'testDates',
+          date: `2023-03-06`,
+          datetime: '2023-03-06T15:47:00.000Z',
+          timestamp: '2023-03-06T15:47:00.000Z'
+        })
+      ]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      expect(endRowCount - startRowCount).toEqual(1);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+
+  test(
+    'table not in sync rules',
+    binlogStreamTest(factory, async (context) => {
+      const { connectionManager } = context;
+      await context.updateSyncRules(BASIC_SYNC_RULES);
+
+      await connectionManager.query(`CREATE TABLE test_donotsync (id CHAR(36) PRIMARY KEY, description text)`);
+
+      await context.replicateSnapshot();
+
+      const startRowCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const startTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      context.startStreaming();
+
+      await connectionManager.query(`INSERT INTO test_donotsync(id, description) VALUES('${uuid()}','test1')`);
+      const data = await context.getBucketData('global[]');
+
+      expect(data).toMatchObject([]);
+      const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+      const endTxCount =
+        (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+
+      // There was a transaction, but we should not replicate any actual data
+      expect(endRowCount - startRowCount).toEqual(0);
+      expect(endTxCount - startTxCount).toEqual(1);
+    })
+  );
+}

package/test/src/BinlogStreamUtils.ts

@@ -0,0 +1,157 @@
+import {
+  ActiveCheckpoint,
+  BucketStorageFactory,
+  OpId,
+  OplogEntry,
+  SyncRulesBucketStorage
+} from '@powersync/service-core';
+import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+import { fromAsync } from '@core-tests/stream_utils.js';
+import { BinLogStream, BinLogStreamOptions } from '@module/replication/BinLogStream.js';
+import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import mysqlPromise from 'mysql2/promise';
+import { readExecutedGtid } from '@module/common/read-executed-gtid.js';
+import { logger } from '@powersync/lib-services-framework';
+
+/**
+ * Tests operating on the binlog stream need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps a test in a function that configures all the context, and tears it down afterward.
+ */
+export function binlogStreamTest(
+  factory: () => Promise<BucketStorageFactory>,
+  test: (context: BinlogStreamTestContext) => Promise<void>
+): () => Promise<void> {
+  return async () => {
+    const f = await factory();
+    const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
+
+    const connection = await connectionManager.getConnection();
+    await clearTestDb(connection);
+    connection.release();
+    const context = new BinlogStreamTestContext(f, connectionManager);
+    try {
+      await test(context);
+    } finally {
+      await context.dispose();
+    }
+  };
+}
+
+export class BinlogStreamTestContext {
+  private _binlogStream?: BinLogStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+  private replicationDone = false;
+
+  constructor(
+    public factory: BucketStorageFactory,
+    public connectionManager: MySQLConnectionManager
+  ) {}
+
+  async dispose() {
+    this.abortController.abort();
+    await this.streamPromise;
+    await this.connectionManager.end();
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  async updateSyncRules(content: string): Promise<SyncRulesBucketStorage> {
+    const syncRules = await this.factory.updateSyncRules({ content: content });
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  get binlogStream(): BinLogStream {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._binlogStream) {
+      return this._binlogStream;
+    }
+    const options: BinLogStreamOptions = {
+      storage: this.storage,
+      connections: this.connectionManager,
+      abortSignal: this.abortController.signal
+    };
+    this._binlogStream = new BinLogStream(options);
+    return this._binlogStream!;
+  }
+
+  async replicateSnapshot() {
+    await this.binlogStream.initReplication();
+    this.replicationDone = true;
+  }
+
+  startStreaming() {
+    if (!this.replicationDone) {
+      throw new Error('Call replicateSnapshot() before startStreaming()');
+    }
+    this.streamPromise = this.binlogStream.streamChanges();
+  }
+
+  async getCheckpoint(options?: { timeout?: number }): Promise<string> {
+    const connection = await this.connectionManager.getConnection();
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(connection, this.factory, { timeout: options?.timeout ?? 60_000 }),
+      this.streamPromise
+    ]);
+    connection.release();
+    if (typeof checkpoint == undefined) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint as string;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+    const checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>(Object.entries(buckets));
+    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  async getBucketData(bucket: string, start = '0', options?: { timeout?: number }): Promise<OplogEntry[]> {
+    const checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, string>([[bucket, start]]);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await fromAsync(batch);
+    return batches[0]?.batch.data ?? [];
+  }
+}
+
+export async function getClientCheckpoint(
+  connection: mysqlPromise.Connection,
+  bucketStorage: BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+  const gtid = await readExecutedGtid(connection);
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+  let lastCp: ActiveCheckpoint | null = null;
+
+  logger.info('Expected Checkpoint: ' + gtid.comparable);
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    lastCp = cp;
+    //logger.info('Last Checkpoint: ' + lastCp.lsn);
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn && cp.lsn >= gtid.comparable) {
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error(`Timeout while waiting for checkpoint ${gtid.comparable}. Last checkpoint: ${lastCp?.lsn}`);
+}

package/test/src/env.ts

@@ -1,7 +1,7 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
-  MYSQL_TEST_URI: utils.type.string.default('mysql://myuser:mypassword@localhost:3306/mydatabase'),
+  MYSQL_TEST_URI: utils.type.string.default('mysql://root:mypassword@localhost:3306/mydatabase'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')
 });