@powersync/service-module-mysql 0.1.6 → 0.1.8

package/test/src/BinlogStreamUtils.ts

@@ -1,44 +1,26 @@
+import { readExecutedGtid } from '@module/common/read-executed-gtid.js';
+import { BinLogStream, BinLogStreamOptions } from '@module/replication/BinLogStream.js';
+import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import { logger } from '@powersync/lib-services-framework';
 import {
   ActiveCheckpoint,
   BucketStorageFactory,
   OpId,
   OplogEntry,
+  storage,
   SyncRulesBucketStorage
 } from '@powersync/service-core';
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
-import { fromAsync } from '@core-tests/stream_utils.js';
-import { BinLogStream, BinLogStreamOptions } from '@module/replication/BinLogStream.js';
-import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
+import { test_utils } from '@powersync/service-core-tests';
 import mysqlPromise from 'mysql2/promise';
-import { readExecutedGtid } from '@module/common/read-executed-gtid.js';
-import { logger } from '@powersync/lib-services-framework';
+import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 
 /**
  * Tests operating on the binlog stream need to configure the stream and manage asynchronous
  * replication, which gets a little tricky.
  *
- * This wraps a test in a function that configures all the context, and tears it down afterward.
+ * This wraps all the context required for testing, and tears it down afterward
+ * by using `await using`.
  */
-export function binlogStreamTest(
-  factory: () => Promise<BucketStorageFactory>,
-  test: (context: BinlogStreamTestContext) => Promise<void>
-): () => Promise<void> {
-  return async () => {
-    const f = await factory();
-    const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
-
-    const connection = await connectionManager.getConnection();
-    await clearTestDb(connection);
-    connection.release();
-    const context = new BinlogStreamTestContext(f, connectionManager);
-    try {
-      await test(context);
-    } finally {
-      await context.dispose();
-    }
-  };
-}
-
 export class BinlogStreamTestContext {
   private _binlogStream?: BinLogStream;
   private abortController = new AbortController();
@@ -46,6 +28,18 @@ export class BinlogStreamTestContext {
   public storage?: SyncRulesBucketStorage;
   private replicationDone = false;
 
+  static async open(factory: storage.TestStorageFactory, options?: { doNotClear?: boolean }) {
+    const f = await factory({ doNotClear: options?.doNotClear });
+    const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
+
+    if (!options?.doNotClear) {
+      const connection = await connectionManager.getConnection();
+      await clearTestDb(connection);
+      connection.release();
+    }
+    return new BinlogStreamTestContext(f, connectionManager);
+  }
+
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: MySQLConnectionManager
@@ -57,6 +51,10 @@ export class BinlogStreamTestContext {
     await this.connectionManager.end();
   }
 
+  [Symbol.asyncDispose]() {
+    return this.dispose();
+  }
+
   get connectionTag() {
     return this.connectionManager.connectionTag;
   }
@@ -67,6 +65,27 @@ export class BinlogStreamTestContext {
     return this.storage!;
   }
 
+  async loadNextSyncRules() {
+    const syncRules = await this.factory.getNextSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Next sync rules not available`);
+    }
+
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  async loadActiveSyncRules() {
+    const syncRules = await this.factory.getActiveSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Active sync rules not available`);
+    }
+
+    this.storage = this.factory.getInstance(syncRules);
+    this.replicationDone = true;
+    return this.storage!;
+  }
+
   get binlogStream(): BinLogStream {
     if (this.storage == null) {
       throw new Error('updateSyncRules() first');
@@ -113,14 +132,14 @@ export class BinlogStreamTestContext {
   async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
     const checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>(Object.entries(buckets));
-    return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+    return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
   async getBucketData(bucket: string, start = '0', options?: { timeout?: number }): Promise<OplogEntry[]> {
     const checkpoint = await this.getCheckpoint(options);
     const map = new Map<string, string>([[bucket, start]]);
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
-    const batches = await fromAsync(batch);
+    const batches = await test_utils.fromAsync(batch);
     return batches[0]?.batch.data ?? [];
   }
 }

package/test/src/env.ts

@@ -2,6 +2,7 @@ import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
   MYSQL_TEST_URI: utils.type.string.default('mysql://root:mypassword@localhost:3306/mydatabase'),
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false')
 });

package/test/src/mysql-to-sqlite.test.ts

@@ -200,6 +200,7 @@ INSERT INTO test_data (
 
   test('Date types mappings', async () => {
     await setupTable();
+    // Timezone offset is set on the pool to +00:00
     await connectionManager.query(`
         INSERT INTO test_data(date_col, datetime_col, timestamp_col, time_col, year_col)
         VALUES('2023-03-06', '2023-03-06 15:47', '2023-03-06 15:47', '15:47:00', '2023');
@@ -222,23 +223,40 @@ INSERT INTO test_data (
   test('Date types edge cases mappings', async () => {
     await setupTable();
 
-    await connectionManager.query(`INSERT INTO test_data(timestamp_col) VALUES('1970-01-01 00:00:01')`);
-    await connectionManager.query(`INSERT INTO test_data(timestamp_col) VALUES('2038-01-19 03:14:07.499')`);
-    await connectionManager.query(`INSERT INTO test_data(datetime_col) VALUES('1000-01-01 00:00:00')`);
-    await connectionManager.query(`INSERT INTO test_data(datetime_col) VALUES('9999-12-31 23:59:59.499')`);
-
-    const databaseRows = await getDatabaseRows(connectionManager, 'test_data');
-    const replicatedRows = await getReplicatedRows(4);
-    const expectedResults = [
-      { timestamp_col: '1970-01-01T00:00:01.000Z' },
-      { timestamp_col: '2038-01-19T03:14:07.499Z' },
-      { datetime_col: '1000-01-01T00:00:00.000Z' },
-      { datetime_col: '9999-12-31T23:59:59.499Z' }
-    ];
-
-    for (let i = 0; i < expectedResults.length; i++) {
-      expect(databaseRows[i]).toMatchObject(expectedResults[i]);
-      expect(replicatedRows[i]).toMatchObject(expectedResults[i]);
+    const connection = await connectionManager.getConnection();
+    try {
+      // Disable strict mode, to allow dates such as '2024-00-00'.
+      await connection.query(`SET SESSION sql_mode=''`);
+      await connection.query(`SET SESSION time_zone='+00:00'`);
+
+      await connection.query(`INSERT INTO test_data(timestamp_col) VALUES('1970-01-01 00:00:01')`);
+      await connection.query(`INSERT INTO test_data(timestamp_col) VALUES('2038-01-19 03:14:07.499')`);
+      await connection.query(`INSERT INTO test_data(datetime_col) VALUES('1000-01-01 00:00:00')`);
+      await connection.query(`INSERT INTO test_data(datetime_col) VALUES('9999-12-31 23:59:59.499')`);
+      await connection.query(`INSERT INTO test_data(datetime_col) VALUES('0000-00-00 00:00:00')`);
+      await connection.query(`INSERT INTO test_data(datetime_col) VALUES('2024-00-00 00:00:00')`);
+      // TODO: This has a mismatch between querying directly and with Zongji.
+      // await connection.query(`INSERT INTO test_data(date_col) VALUES('2024-00-00')`);
+
+      const expectedResults = [
+        { timestamp_col: '1970-01-01T00:00:01.000Z' },
+        { timestamp_col: '2038-01-19T03:14:07.499Z' },
+        { datetime_col: '1000-01-01T00:00:00.000Z' },
+        { datetime_col: '9999-12-31T23:59:59.499Z' },
+        { datetime_col: null },
+        { datetime_col: null }
+        // { date_col: '2023-11-30' } or { date_col: null }?
+      ];
+
+      const databaseRows = await getDatabaseRows(connectionManager, 'test_data');
+      const replicatedRows = await getReplicatedRows(expectedResults.length);
+
+      for (let i = 0; i < expectedResults.length; i++) {
+        expect(databaseRows[i]).toMatchObject(expectedResults[i]);
+        expect(replicatedRows[i]).toMatchObject(expectedResults[i]);
+      }
+    } finally {
+      connection.release;
     }
   });
 
@@ -282,8 +300,7 @@ async function getReplicatedRows(expectedTransactionsCount?: number): Promise<Sq
   const zongji = new ZongJi({
     host: TEST_CONNECTION_OPTIONS.hostname,
     user: TEST_CONNECTION_OPTIONS.username,
-    password: TEST_CONNECTION_OPTIONS.password,
-    timeZone: 'Z' // Ensure no auto timezone manipulation of the dates occur
+    password: TEST_CONNECTION_OPTIONS.password
   });
 
   const completionPromise = new Promise<SqliteRow[]>((resolve, reject) => {

package/test/src/setup.ts

@@ -1,7 +1,9 @@
 import { container } from '@powersync/lib-services-framework';
+import { test_utils } from '@powersync/service-core-tests';
 import { beforeAll } from 'vitest';
 
-beforeAll(() => {
+beforeAll(async () => {
   // Executes for every test file
   container.registerDefaults();
+  await test_utils.initMetrics();
 });

package/test/src/util.ts

@@ -1,9 +1,8 @@
 import * as types from '@module/types/types.js';
-import { BucketStorageFactory, Metrics, MongoBucketStorage } from '@powersync/service-core';
-import { env } from './env.js';
-import mysqlPromise from 'mysql2/promise';
-import { connectMongo } from '@core-tests/util.js';
 import { getMySQLVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js';
+import * as mongo_storage from '@powersync/service-module-mongodb-storage';
+import mysqlPromise from 'mysql2/promise';
+import { env } from './env.js';
 
 export const TEST_URI = env.MYSQL_TEST_URI;
 
@@ -12,28 +11,10 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   uri: TEST_URI
 });
 
-// The metrics need to be initialized before they can be used
-await Metrics.initialise({
-  disable_telemetry_sharing: true,
-  powersync_instance_id: 'test',
-  internal_metrics_endpoint: 'unused.for.tests.com'
+export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({
+  url: env.MONGO_TEST_URL,
+  isCI: env.CI
 });
-Metrics.getInstance().resetCounters();
-
-export type StorageFactory = () => Promise<BucketStorageFactory>;
-
-export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => {
-  const db = await connectMongo();
-
-  // None of the tests insert data into this collection, so it was never created
-  if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-    await db.db.createCollection('bucket_parameters');
-  }
-
-  await db.clear();
-
-  return new MongoBucketStorage(db, { slot_name_prefix: 'test_' });
-};
 
 export async function clearTestDb(connection: mysqlPromise.Connection) {
   const version = await getMySQLVersion(connection);