@powersync/service-module-mysql 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/test/src/BinLogListener_queue.test.ts

@@ -0,0 +1,62 @@
+import { ReplicatedGTID } from '@module/common/ReplicatedGTID.js';
+import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import { BinLogListener } from '@module/replication/zongji/BinLogListener.js';
+import { ZongJi } from '@powersync/mysql-zongji';
+import { EventEmitter } from 'node:events';
+import { setImmediate } from 'node:timers/promises';
+import { expect, test, vi } from 'vitest';
+import { TEST_CONNECTION_OPTIONS, TestBinLogEventHandler } from './util.js';
+
+test.each(['before', 'after'] as const)('restarts when the queue drains %s Zongji stops', async (drainOrder) => {
+  const manager = new MySQLConnectionManager({ ...TEST_CONNECTION_OPTIONS, binlog_queue_memory_limit: 1 }, {});
+  const stopStarted = Promise.withResolvers<void>();
+  const processEvent = Promise.withResolvers<void>();
+  // Control the external listener's stop acknowledgement independently of the
+  // real processing queue: MySQL can acknowledge KILL after the queue drains.
+  const zongji = Object.assign(new EventEmitter(), {
+    stopped: false,
+    ctrlConnection: {},
+    stop() {
+      this.stopped = true;
+      stopStarted.resolve();
+    }
+  });
+  vi.spyOn(manager, 'createBinlogListener').mockImplementation(() => zongji as unknown as ZongJi);
+  const handler = new TestBinLogEventHandler();
+  vi.spyOn(handler, 'onKeepAlive').mockImplementation(() => processEvent.promise);
+  const listener = new BinLogListener({
+    connectionManager: manager,
+    eventHandler: handler,
+    sourceTables: [],
+    serverId: 1,
+    activeServerUuid: 'test',
+    startGTID: ReplicatedGTID.ZERO('test')
+  });
+  const restart = vi.spyOn(listener, 'start').mockResolvedValue();
+
+  try {
+    const drained = listener.processingQueue.drain();
+    zongji.emit('binlog', { getEventName: () => 'heartbeat', size: 1024 * 1024 });
+    await stopStarted.promise;
+    if (drainOrder === 'before') {
+      processEvent.resolve();
+      await drained;
+      expect(listener.processingQueue.idle()).toBe(true);
+      zongji.emit('stopped');
+    } else {
+      zongji.emit('stopped');
+      // Let stopZongji return while the queue worker is still blocked.
+      await setImmediate();
+      expect(listener.processingQueue.running()).toBe(1);
+      expect(restart).not.toHaveBeenCalled();
+      processEvent.resolve();
+    }
+    await vi.waitFor(() => expect(restart).toHaveBeenCalledExactlyOnceWith(true));
+    expect(listener.queueMemoryUsage).toBe(0);
+  } finally {
+    processEvent.resolve();
+    zongji.emit('stopped');
+    await listener.stop();
+    await manager.end();
+  }
+});

package/test/src/BinlogStreamUtils.ts

@@ -15,7 +15,7 @@ import {
   SyncRulesBucketStorage,
   updateSyncRulesFromYaml
 } from '@powersync/service-core';
-import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, getTestStorage, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import mysqlPromise from 'mysql2/promise';
 import timers from 'timers/promises';
 import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
@@ -75,7 +75,7 @@ export class BinlogStreamTestContext {
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION })
     );
     this.syncRulesContent = replicationStream.syncConfigContent[0];
-    this.storage = this.factory.getInstance(replicationStream);
+    this.storage = await getTestStorage(this.factory, replicationStream);
     return this.storage!;
   }
 
@@ -86,7 +86,7 @@ export class BinlogStreamTestContext {
     }
 
     this.syncRulesContent = syncConfig.content;
-    this.storage = syncConfig.storage;
+    this.storage = await getTestStorage(this.factory, syncConfig.replicationStream);
     return this.storage!;
   }
 
@@ -97,7 +97,7 @@ export class BinlogStreamTestContext {
     }
 
     this.syncRulesContent = syncConfig.content;
-    this.storage = syncConfig.storage;
+    this.storage = await getTestStorage(this.factory, syncConfig.replicationStream);
     this.replicationDone = true;
     return this.storage!;
   }

package/test/src/ReplicatedGTID.test.ts

@@ -0,0 +1,138 @@
+import { ReplicatedGTID } from '@module/common/ReplicatedGTID.js';
+import * as uuid from 'uuid';
+import { describe, expect, test } from 'vitest';
+
+describe('ReplicatedGTID', () => {
+  const SERVER_UUID = 'a7d0ff7b-0c0e-11f0-8b38-566fbaa00004';
+  const POSITION = { filename: 'binlog.000042', offset: 1234 };
+
+  describe('single GTID', () => {
+    test('exposes its raw value, server UUID, and binlog position', () => {
+      const gtid = new ReplicatedGTID({
+        rawGtid: `${SERVER_UUID}:5`,
+        position: POSITION
+      });
+
+      expect(gtid.raw).toEqual(`${SERVER_UUID}:5`);
+      expect(gtid.serverUuid).toEqual(SERVER_UUID);
+      expect(gtid.position).toEqual(POSITION);
+    });
+
+    test('formats a comparable LSN using the transaction id', () => {
+      const gtid = new ReplicatedGTID({
+        rawGtid: `${SERVER_UUID}:17`,
+        position: POSITION
+      });
+
+      expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
+      expect(gtid.toString()).toEqual(gtid.comparable);
+    });
+
+    test('normalizes surrounding whitespace', () => {
+      const gtid = new ReplicatedGTID({
+        rawGtid: ` \n\t${SERVER_UUID}:17 \r\n`,
+        position: POSITION
+      });
+
+      expect(gtid.raw).toEqual(`${SERVER_UUID}:17`);
+      expect(gtid.serverUuid).toEqual(SERVER_UUID);
+      expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
+    });
+
+    test('keeps the ZERO GTID format stable', () => {
+      expect(ReplicatedGTID.ZERO(SERVER_UUID).raw).toEqual(`${SERVER_UUID}:0`);
+      expect(ReplicatedGTID.ZERO(SERVER_UUID).comparable).toEqual(`0000000000000000|${SERVER_UUID}:0||0`);
+    });
+  });
+
+  describe('validation', () => {
+    test.each([
+      ['', 'missing server UUID and transaction id'],
+      [SERVER_UUID, 'missing transaction id'],
+      [`${SERVER_UUID}:`, 'empty transaction id'],
+      [`:${17}`, 'empty server UUID'],
+      [`${SERVER_UUID}:1-17`, 'transaction interval'],
+      [`${SERVER_UUID}:1:17`, 'multiple transaction components'],
+      [`${SERVER_UUID}:abc`, 'non-numeric transaction id'],
+      [`${SERVER_UUID}:-1`, 'negative transaction id'],
+      [`${SERVER_UUID}:17,another-server:9`, 'comma-separated GTID set'],
+      [`${SERVER_UUID}:17,\nanother-server:9`, 'newline-separated GTID set']
+    ])('rejects %s (%s)', (rawGtid) => {
+      expect(() => new ReplicatedGTID({ rawGtid, position: POSITION })).toThrow();
+    });
+  });
+
+  describe('serialization', () => {
+    test('round-trips a single GTID', () => {
+      const gtid = new ReplicatedGTID({
+        rawGtid: `${SERVER_UUID}:17`,
+        position: POSITION
+      });
+
+      const deserialized = ReplicatedGTID.fromSerialized(gtid.comparable);
+
+      expect(deserialized.raw).toEqual(gtid.raw);
+      expect(deserialized.serverUuid).toEqual(SERVER_UUID);
+      expect(deserialized.position).toEqual(POSITION);
+      expect(deserialized.comparable).toEqual(gtid.comparable);
+    });
+
+    test('throws on malformed serialized GTIDs', () => {
+      expect(() => ReplicatedGTID.fromSerialized('abc')).toThrow('Invalid serialized GTID');
+      expect(() => ReplicatedGTID.fromSerialized(`0000000000000001|${SERVER_UUID}:1|binlog.000001`)).toThrow(
+        'Invalid serialized GTID'
+      );
+      expect(() => ReplicatedGTID.fromSerialized(`0000000000000001|${SERVER_UUID}:1|binlog.000001|notanumber`)).toThrow(
+        'Invalid BinLog offset'
+      );
+    });
+
+    test('rejects a serialized GTID set', () => {
+      const serialized = `0000000000000017|${SERVER_UUID}:1-17|binlog.000042|1234`;
+
+      expect(() => ReplicatedGTID.fromSerialized(serialized)).toThrow('Expected a single transaction id');
+    });
+  });
+
+  describe('binlog events', () => {
+    test('creates a single GTID from a binlog event', () => {
+      const gtid = ReplicatedGTID.fromBinLogEvent({
+        rawGtid: {
+          serverUuid: Buffer.from(uuid.parse(SERVER_UUID)),
+          transactionId: 17
+        },
+        position: POSITION
+      });
+
+      expect(gtid.raw).toEqual(`${SERVER_UUID}:17`);
+      expect(gtid.position).toEqual(POSITION);
+      expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
+    });
+  });
+
+  describe('LSN ordering', () => {
+    test('orders GTIDs from the same server by transaction id', () => {
+      const earlier = new ReplicatedGTID({ rawGtid: `${SERVER_UUID}:9`, position: POSITION });
+      const later = new ReplicatedGTID({ rawGtid: `${SERVER_UUID}:18`, position: POSITION });
+
+      expect(earlier.comparable < later.comparable).toBeTruthy();
+    });
+
+    test('orders LSNs for the same transaction by binlog offset', () => {
+      // The binlog offset is not zero-padded, so lexicographic ordering only holds for
+      // offsets with the same number of digits. This format cannot change while existing
+      // LSNs remain persisted in bucket storage.
+      const rawGtid = `${SERVER_UUID}:18`;
+      const transactionStart = new ReplicatedGTID({
+        rawGtid,
+        position: { filename: 'binlog.000042', offset: 157 }
+      });
+      const transactionEnd = new ReplicatedGTID({
+        rawGtid,
+        position: { filename: 'binlog.000042', offset: 300 }
+      });
+
+      expect(transactionStart.comparable < transactionEnd.comparable).toBeTruthy();
+    });
+  });
+});

package/test/src/check-source-configuration.test.ts

@@ -0,0 +1,70 @@
+import { checkSourceConfiguration } from '@module/common/check-source-configuration.js';
+import { describe, expect, test } from 'vitest';
+import { createMockMySQLConnection } from './util.js';
+
+describe('checkSourceConfiguration', () => {
+  test('accepts a primary MySQL server', async () => {
+    const { connection, query } = createConnection({ version: '8.4.0', replicaStatuses: [] });
+
+    await expect(checkSourceConfiguration(connection)).resolves.toEqual([]);
+    expect(query).toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
+  });
+
+  test('rejects a replica on MySQL 8.0.22 and later', async () => {
+    const { connection, query } = createConnection({
+      version: '8.0.22',
+      replicaStatuses: [{ Channel_Name: '' }]
+    });
+
+    await expect(checkSourceConfiguration(connection)).resolves.toContain(
+      'Connecting PowerSync to a MySQL replica is not supported. Please connect PowerSync directly to the primary server.'
+    );
+    expect(query).toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
+    expect(query).not.toHaveBeenCalledWith('SHOW SLAVE STATUS', []);
+  });
+
+  test('uses legacy replica-status syntax before MySQL 8.0.22', async () => {
+    const { connection, query } = createConnection({
+      version: '5.7.44',
+      replicaStatuses: [{ Channel_Name: '' }]
+    });
+
+    await expect(checkSourceConfiguration(connection)).resolves.toContain(
+      'Connecting PowerSync to a MySQL replica is not supported. Please connect PowerSync directly to the primary server.'
+    );
+    expect(query).toHaveBeenCalledWith('SHOW SLAVE STATUS', []);
+    expect(query).not.toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
+  });
+
+  function createConnection(options: { version: string; replicaStatuses: Record<string, unknown>[] }) {
+    return createMockMySQLConnection(async (sql) => {
+      switch (sql.trim()) {
+        case 'SELECT VERSION() as version':
+          return [[{ version: options.version }], []];
+        case 'SHOW REPLICA STATUS':
+        case 'SHOW SLAVE STATUS':
+          return [options.replicaStatuses, []];
+        case "SHOW VARIABLES LIKE 'binlog_format';":
+          return [[{ Value: 'ROW' }], []];
+        case "SHOW GLOBAL VARIABLES LIKE 'binlog_row_image';":
+          return [[{ Value: 'FULL' }], []];
+        default:
+          if (sql.includes('@@GLOBAL.gtid_mode AS gtid_mode')) {
+            return [
+              [
+                {
+                  gtid_mode: 'ON',
+                  log_bin: 1,
+                  server_id: 1,
+                  binlog_file: '/var/lib/mysql/binlog',
+                  binlog_index_file: '/var/lib/mysql/binlog.index'
+                }
+              ],
+              []
+            ];
+          }
+          throw new Error(`Unexpected query: ${sql}`);
+      }
+    });
+  }
+});

package/test/src/mysql-utils.test.ts

@@ -1,4 +1,5 @@
-import { isVersionAtLeast } from '@module/utils/mysql-utils.js';
+import * as types from '@module/types/types.js';
+import { createPool, isVersionAtLeast, TCP_KEEPALIVE_INITIAL_DELAY } from '@module/utils/mysql-utils.js';
 import { describe, expect, test } from 'vitest';
 
 describe('MySQL Utility Tests', () => {
@@ -14,4 +15,21 @@ describe('MySQL Utility Tests', () => {
     expect(isVersionAtLeast(olderVersion, '8.0')).toBeFalsy();
     expect(isVersionAtLeast(improperSemver, '5.7')).toBeTruthy();
   });
+
+  test('Pool connections are configured with a TCP keepalive initial delay', async () => {
+    // mysql2 enables keepalive by default, but without an initial delay the OS default of
+    // 7200 seconds applies, which is too late for common 3600 second firewall idle timeouts.
+    const config = types.normalizeConnectionConfig({
+      type: 'mysql',
+      uri: 'mysql://root:password@localhost:3306/mydatabase'
+    });
+    // The pool is lazy, so no connection is made here.
+    const pool = createPool(config);
+    const { connectionConfig } = (pool as unknown as { config: { connectionConfig: Record<string, unknown> } }).config;
+
+    expect(connectionConfig.enableKeepAlive).toBe(true);
+    expect(connectionConfig.keepAliveInitialDelay).toBe(TCP_KEEPALIVE_INITIAL_DELAY);
+
+    await pool.promise().end();
+  });
 });

package/test/src/read-executed-gtid.test.ts

@@ -0,0 +1,188 @@
+import { ReplicatedGTID } from '@module/common/ReplicatedGTID.js';
+import {
+  getLatestActiveGtid,
+  isGtidPositionStillAvailable,
+  readExecutedGtid
+} from '@module/common/read-executed-gtid.js';
+import { describe, expect, test } from 'vitest';
+import { createMockMySQLConnection } from './util.js';
+
+describe('read-executed-gtid', () => {
+  const ACTIVE_SERVER_UUID = 'a7d0ff7b-0c0e-11f0-8b38-566fbaa00004';
+  const STALE_SERVER_UUID = '314306f3-ff7b-11ef-a0e0-566fbaa00002';
+
+  describe('getLatestActiveGtid', () => {
+    test('returns the highest transaction id for the active server', async () => {
+      const gtid = await getLatestActiveGtid(
+        [`${STALE_SERVER_UUID}:1-1000`, `\n${ACTIVE_SERVER_UUID}:1-5:9:12-17`],
+        ACTIVE_SERVER_UUID
+      );
+
+      expect(gtid).toEqual(`${ACTIVE_SERVER_UUID}:17`);
+    });
+
+    test('supports a bare transaction id', async () => {
+      const gtid = await getLatestActiveGtid([`${ACTIVE_SERVER_UUID}:42`], ACTIVE_SERVER_UUID);
+
+      expect(gtid).toEqual(`${ACTIVE_SERVER_UUID}:42`);
+    });
+
+    test('returns the active server ZERO GTID when it is absent from the GTID sets', async () => {
+      await expect(getLatestActiveGtid([`${STALE_SERVER_UUID}:1-1000`], ACTIVE_SERVER_UUID)).resolves.toEqual(
+        `${ACTIVE_SERVER_UUID}:0`
+      );
+    });
+  });
+
+  describe('readExecutedGtid', () => {
+    test('reads binary log status on MySQL 8.4 and selects the active server GTID', async () => {
+      const { connection, query } = createConnection({
+        version: '8.4.0',
+        executedGtidSet: `${STALE_SERVER_UUID}:1-1000,\n${ACTIVE_SERVER_UUID}:1-5:11-18`
+      });
+
+      const gtid = await readExecutedGtid(connection);
+
+      expect(gtid.raw).toEqual(`${ACTIVE_SERVER_UUID}:18`);
+      expect(gtid.position).toEqual({ filename: 'binlog.000042', offset: 1234 });
+      expect(query).toHaveBeenCalledWith('SHOW BINARY LOG STATUS', []);
+      expect(query).not.toHaveBeenCalledWith('SHOW MASTER STATUS', []);
+    });
+
+    test('reads master status on MySQL versions before 8.4', async () => {
+      const { connection, query } = createConnection({
+        version: '8.0.40',
+        executedGtidSet: `${ACTIVE_SERVER_UUID}:1-17`
+      });
+
+      const gtid = await readExecutedGtid(connection);
+
+      expect(gtid.raw).toEqual(`${ACTIVE_SERVER_UUID}:17`);
+      expect(query).toHaveBeenCalledWith('SHOW MASTER STATUS', []);
+      expect(query).not.toHaveBeenCalledWith('SHOW BINARY LOG STATUS', []);
+    });
+
+    test('returns the active server ZERO GTID when no transactions have executed', async () => {
+      const { connection } = createConnection({
+        version: '8.4.0',
+        executedGtidSet: ' \n\t '
+      });
+
+      const gtid = await readExecutedGtid(connection);
+
+      expect(gtid.raw).toEqual(`${ACTIVE_SERVER_UUID}:0`);
+      expect(gtid.position).toEqual({ filename: 'binlog.000042', offset: 1234 });
+      expect(gtid.comparable).toEqual(`0000000000000000|${ACTIVE_SERVER_UUID}:0|binlog.000042|1234`);
+    });
+
+    test('uses the active server ZERO GTID at the current position when only historical UUIDs exist', async () => {
+      const { connection } = createConnection({
+        version: '8.4.0',
+        executedGtidSet: `${STALE_SERVER_UUID}:1-1000`
+      });
+
+      const gtid = await readExecutedGtid(connection);
+
+      expect(gtid.raw).toEqual(`${ACTIVE_SERVER_UUID}:0`);
+      expect(gtid.position).toEqual({ filename: 'binlog.000042', offset: 1234 });
+      expect(gtid.comparable).toEqual(`0000000000000000|${ACTIVE_SERVER_UUID}:0|binlog.000042|1234`);
+    });
+  });
+
+  describe('isGtidPositionStillAvailable', () => {
+    const RESUME_GTID = new ReplicatedGTID({
+      rawGtid: `${ACTIVE_SERVER_UUID}:17`,
+      position: { filename: 'binlog.000042', offset: 1234 }
+    });
+
+    test('returns true when the GTID is executed and its binlog coordinate is available', async () => {
+      const { connection, query } = createResumeCheckConnection({
+        isExecuted: 1,
+        logFiles: [{ Log_name: 'binlog.000042', File_size: 2000 }]
+      });
+
+      await expect(isGtidPositionStillAvailable(connection, RESUME_GTID)).resolves.toBe(true);
+      expect(query).toHaveBeenCalledWith('SELECT GTID_SUBSET(?, @@GLOBAL.gtid_executed) AS is_executed', [
+        RESUME_GTID.raw
+      ]);
+    });
+
+    test('returns false when the GTID is absent after a source rewind', async () => {
+      const { connection } = createResumeCheckConnection({
+        isExecuted: 0,
+        logFiles: [{ Log_name: 'binlog.000042', File_size: 2000 }]
+      });
+
+      await expect(isGtidPositionStillAvailable(connection, RESUME_GTID)).resolves.toBe(false);
+    });
+
+    test('validates the synthetic ZERO GTID using only its binlog coordinate', async () => {
+      const zeroGtid = new ReplicatedGTID({
+        rawGtid: `${ACTIVE_SERVER_UUID}:0`,
+        position: { filename: 'binlog.000042', offset: 1234 }
+      });
+      const { connection, query } = createResumeCheckConnection({
+        isExecuted: 0,
+        logFiles: [{ Log_name: 'binlog.000042', File_size: 2000 }]
+      });
+
+      await expect(isGtidPositionStillAvailable(connection, zeroGtid)).resolves.toBe(true);
+      expect(query).not.toHaveBeenCalledWith('SELECT GTID_SUBSET(?, @@GLOBAL.gtid_executed) AS is_executed', [
+        zeroGtid.raw
+      ]);
+    });
+
+    test.each([
+      [[], 'the binlog file is absent'],
+      [[{ Log_name: 'binlog.000042', File_size: 1000 }], 'the stored offset is past the end of the binlog']
+    ])('returns false when %s (%s)', async (logFiles) => {
+      const { connection, query } = createResumeCheckConnection({ isExecuted: 1, logFiles });
+
+      await expect(isGtidPositionStillAvailable(connection, RESUME_GTID)).resolves.toBe(false);
+      expect(query).not.toHaveBeenCalledWith('SELECT GTID_SUBSET(?, @@GLOBAL.gtid_executed) AS is_executed', [
+        RESUME_GTID.raw
+      ]);
+    });
+  });
+
+  function createConnection(options: { version: string; executedGtidSet: string }) {
+    return createMockMySQLConnection(async (sql) => {
+      switch (sql) {
+        case 'SELECT VERSION() as version':
+          return [[{ version: options.version }], []];
+        case 'SHOW BINARY LOG STATUS':
+        case 'SHOW MASTER STATUS':
+          return [
+            [
+              {
+                File: 'binlog.000042',
+                Position: '1234',
+                Executed_Gtid_Set: options.executedGtidSet
+              }
+            ],
+            []
+          ];
+        case 'SELECT @@server_uuid AS server_uuid':
+          return [[{ server_uuid: ACTIVE_SERVER_UUID }], []];
+        default:
+          throw new Error(`Unexpected query: ${sql}`);
+      }
+    });
+  }
+
+  function createResumeCheckConnection(options: {
+    isExecuted: number;
+    logFiles: { Log_name: string; File_size: number }[];
+  }) {
+    return createMockMySQLConnection(async (sql) => {
+      switch (sql) {
+        case 'SHOW BINARY LOGS;':
+          return [options.logFiles, []];
+        case 'SELECT GTID_SUBSET(?, @@GLOBAL.gtid_executed) AS is_executed':
+          return [[{ is_executed: options.isExecuted }], []];
+        default:
+          throw new Error(`Unexpected query: ${sql}`);
+      }
+    });
+  }
+});