@powersync/service-module-mysql 0.12.5 → 0.14.0

package/package.json

@@ -1,25 +1,15 @@
 {
   "name": "@powersync/service-module-mysql",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "types": "dist/index.d.ts",
-  "version": "0.12.5",
+  "version": "0.14.0",
   "license": "FSL-1.1-ALv2",
-  "main": "dist/index.js",
   "type": "module",
   "publishConfig": {
     "access": "public"
   },
   "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./dist/index.js",
-      "default": "./dist/index.js"
-    },
-    "./types": {
-      "import": "./dist/types/types.js",
-      "require": "./dist/types/types.js",
-      "default": "./dist/types/types.js"
-    }
+    ".": "./dist/index.js",
+    "./types": "./dist/types/types.js"
   },
   "dependencies": {
     "@powersync/mysql-zongji": "^0.6.0",
@@ -30,18 +20,18 @@
     "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^14.0.0",
-    "@powersync/lib-services-framework": "0.9.4",
-    "@powersync/service-core": "1.21.0",
-    "@powersync/service-sync-rules": "0.36.0",
-    "@powersync/service-types": "0.15.2",
+    "@powersync/lib-services-framework": "0.9.6",
+    "@powersync/service-core": "1.23.0",
+    "@powersync/service-sync-rules": "0.38.0",
+    "@powersync/service-types": "0.16.0",
     "@powersync/service-jsonbig": "0.17.13"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/semver": "^7.7.1",
-    "@powersync/service-core-tests": "0.15.5",
-    "@powersync/service-module-mongodb-storage": "0.16.0",
-    "@powersync/service-module-postgres-storage": "0.14.0"
+    "@powersync/service-core-tests": "0.17.0",
+    "@powersync/service-module-mongodb-storage": "0.18.0",
+    "@powersync/service-module-postgres-storage": "0.16.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/MySQLRouteAPIAdapter.ts

@@ -1,4 +1,4 @@
-import { api, ParseSyncRulesOptions, ReplicationHeadCallback, storage } from '@powersync/service-core';
+import { api, ParseSyncConfigOptions, ReplicationHeadCallback } from '@powersync/service-core';
 
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
@@ -29,7 +29,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     return this.config;
   }
 
-  getParseSyncRulesOptions(): ParseSyncRulesOptions {
+  getParseSyncRulesOptions(): ParseSyncConfigOptions {
     return {
       // In MySQL Schema and Database are the same thing. There is no default database
       defaultSchema: this.config.database
@@ -220,20 +220,16 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     }
 
     const idColumns = idColumnsResult?.columns ?? [];
-    const sourceTable = new storage.SourceTable({
-      id: '', // not used
+    const ref: sync_rules.SourceTableRef = {
       connectionTag: this.config.tag,
-      objectId: tableName,
-      schema: schema,
-      name: tableName,
-      replicaIdColumns: idColumns,
-      snapshotComplete: true
-    });
-    const syncData = syncRules.tableSyncsData(sourceTable);
-    const syncParameters = syncRules.tableSyncsParameters(sourceTable);
+      schema,
+      name: tableName
+    };
+    const syncData = syncRules.tableSyncsData(ref);
+    const syncParameters = syncRules.tableSyncsParameters(ref);
 
     if (idColumns.length == 0 && idColumnsError == null) {
-      let message = `No replication id found for ${sourceTable.qualifiedName}. Replica identity: ${idColumnsResult?.identity}.`;
+      let message = `No replication id found for ${mysql_utils.qualifiedMySQLTable(ref)}. Replica identity: ${idColumnsResult?.identity}.`;
       if (idColumnsResult?.identity == 'default') {
         message += ' Configure a primary key on the table.';
       }
@@ -243,7 +239,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     let selectError: service_types.ReplicationError | null = null;
     try {
       await this.retriedQuery({
-        query: `SELECT * FROM ${sourceTable.name} LIMIT 1`
+        query: `SELECT * FROM ${mysql_utils.qualifiedMySQLTable(ref)} LIMIT 1`
       });
     } catch (e) {
       selectError = { level: 'fatal', message: e.message };

package/src/replication/BinLogReplicationJob.ts

@@ -17,8 +17,8 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
     this.connectionFactory = options.connectionFactory;
   }
 
-  get slot_name() {
-    return this.options.storage.slot_name;
+  get replicationStreamName() {
+    return this.options.storage.replicationStreamName;
   }
 
   async keepAlive() {
@@ -37,7 +37,7 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
         // Report the error if relevant, before retrying
         container.reporter.captureException(e, {
           metadata: {
-            replication_slot: this.slot_name
+            replication_slot: this.replicationStreamName
           }
         });
         // This sets the retry delay

package/src/replication/BinLogReplicator.ts

@@ -17,7 +17,7 @@ export class BinLogReplicator extends replication.AbstractReplicator<BinLogRepli
 
   createJob(options: replication.CreateJobOptions): BinLogReplicationJob {
     return new BinLogReplicationJob({
-      id: this.createJobId(options.storage.group_id),
+      id: this.createJobId(options.storage.replicationStreamId),
       storage: options.storage,
       metrics: this.metrics,
       lock: options.lock,

package/src/replication/BinLogStream.ts

@@ -62,7 +62,7 @@ function createTableId(schema: string, tableName: string): string {
 }
 
 export class BinLogStream {
-  private readonly syncRules: sync_rules.HydratedSyncRules;
+  private readonly syncRules: sync_rules.HydratedSyncConfig;
   private readonly groupId: number;
 
   private readonly storage: storage.SyncRulesBucketStorage;
@@ -73,7 +73,7 @@ export class BinLogStream {
 
   private readonly logger: Logger;
 
-  private tableCache = new Map<string | number, storage.SourceTable>();
+  private tableCache = new Map<string | number, storage.SourceTable[]>();
 
   private replicationLag = new ReplicationLagTracker();
 
@@ -82,7 +82,7 @@ export class BinLogStream {
     this.storage = options.storage;
     this.connections = options.connections;
     this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: this.defaultSchema });
-    this.groupId = options.storage.group_id;
+    this.groupId = options.storage.replicationStreamId;
     this.abortSignal = options.abortSignal;
   }
 
@@ -122,16 +122,17 @@ export class BinLogStream {
     return this.connections.databaseName;
   }
 
-  async handleRelation(batch: storage.BucketStorageBatch, entity: storage.SourceEntityDescriptor, snapshot: boolean) {
-    const result = await this.storage.resolveTable({
-      group_id: this.groupId,
+  async handleRelation(
+    batch: storage.BucketStorageBatch,
+    source: storage.SourceEntityDescriptor,
+    snapshot: boolean
+  ): Promise<storage.SourceTable[]> {
+    const result = await batch.resolveTables({
       connection_id: this.connectionId,
-      connection_tag: this.connectionTag,
-      entity_descriptor: entity,
-      sync_rules: this.syncRules
+      source
     });
-    // Since we create the objectId ourselves, this is always defined
-    this.tableCache.set(entity.objectId!, result.table);
+    // Since we create the objectId ourselves, this is always defined.
+    this.tableCache.set(source.objectId!, result.tables);
 
     // Drop conflicting tables. In the MySQL case with ObjectIds created from the table name, renames cannot be detected by the storage.
     await batch.drop(result.dropTables);
@@ -140,11 +141,9 @@ export class BinLogStream {
     // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
     // 2. Snapshot is not done yet, AND:
     // 3. The table is used in sync config.
-    const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
-
-    if (shouldSnapshot) {
-      // Truncate this table in case a previous snapshot was interrupted.
-      await batch.truncate([result.table]);
+    const snapshotCandidates = result.tables.filter((table) => snapshot && !table.snapshotComplete && table.syncAny);
+    if (snapshotCandidates.length > 0) {
+      await batch.truncate(snapshotCandidates);
 
       let gtid: common.ReplicatedGTID;
       // Start the snapshot inside a transaction.
@@ -157,7 +156,9 @@ export class BinLogStream {
         await promiseConnection.query('START TRANSACTION');
         try {
           gtid = await common.readExecutedGtid(promiseConnection);
-          await this.snapshotTable(connection as mysql.Connection, batch, result.table);
+          for (const table of snapshotCandidates) {
+            await this.snapshotTable(connection as mysql.Connection, batch, table);
+          }
           await promiseConnection.query('COMMIT');
         } catch (e) {
           await this.tryRollback(promiseConnection);
@@ -166,11 +167,14 @@ export class BinLogStream {
       } finally {
         connection.release();
       }
-      const [table] = await batch.markTableSnapshotDone([result.table], gtid.comparable);
-      return table;
+      const doneTables = await batch.markTableSnapshotDone(snapshotCandidates, gtid.comparable);
+      const doneTablesById = new Map(doneTables.map((table) => [table.id, table]));
+      const tables = result.tables.map((table) => doneTablesById.get(table.id) ?? table);
+      this.tableCache.set(source.objectId!, tables);
+      return tables;
     }
 
-    return result.table;
+    return result.tables;
   }
 
   async getQualifiedTableNames(
@@ -189,18 +193,19 @@ export class BinLogStream {
     for (const matchedTable of matchedTables) {
       const replicaIdColumns = await this.getReplicaIdColumns(matchedTable, tablePattern.schema);
 
-      const table = await this.handleRelation(
+      const resolvedTables = await this.handleRelation(
         batch,
         {
           name: matchedTable,
           schema: tablePattern.schema,
+          connectionTag: this.connectionTag,
           objectId: createTableId(tablePattern.schema, matchedTable),
           replicaIdColumns: replicaIdColumns
         },
         false
       );
 
-      tables.push(table);
+      tables.push(...resolvedTables);
     }
     return tables;
   }
@@ -210,8 +215,8 @@ export class BinLogStream {
    */
   protected async checkInitialReplicated(): Promise<boolean> {
     const status = await this.storage.getStatus();
-    const lastKnowGTID = status.checkpoint_lsn ? common.ReplicatedGTID.fromSerialized(status.checkpoint_lsn) : null;
-    if (status.snapshot_done && status.checkpoint_lsn) {
+    const lastKnowGTID = status.resumeLsn ? common.ReplicatedGTID.fromSerialized(status.resumeLsn) : null;
+    if (status.snapshotDone) {
       this.logger.info(`Initial replication already done.`);
 
       if (lastKnowGTID) {
@@ -276,7 +281,7 @@ export class BinLogStream {
             }
           }
           const snapshotDoneGtid = await common.readExecutedGtid(promiseConnection);
-          await batch.markAllSnapshotDone(snapshotDoneGtid.comparable);
+          await batch.markSnapshotDone(snapshotDoneGtid.comparable);
           await batch.commit(headGTID.comparable);
         }
       );
@@ -305,11 +310,11 @@ export class BinLogStream {
     batch: storage.BucketStorageBatch,
     table: storage.SourceTable
   ) {
-    this.logger.info(`Replicating ${qualifiedMySQLTable(table)}`);
+    this.logger.info(`Replicating ${qualifiedMySQLTable(table.ref)}`);
     // TODO count rows and log progress at certain batch sizes
 
     // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query
-    const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${qualifiedMySQLTable(table)}`);
+    const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${qualifiedMySQLTable(table.ref)}`);
     const stream = query.stream();
 
     let columns: Map<string, ColumnDescriptor> | undefined = undefined;
@@ -390,26 +395,26 @@ export class BinLogStream {
     }
   }
 
-  private getTable(tableId: string): storage.SourceTable {
-    const table = this.tableCache.get(tableId);
-    if (table == null) {
+  private getTables(tableId: string): storage.SourceTable[] {
+    const tables = this.tableCache.get(tableId);
+    if (tables == null) {
       // We should always receive a replication message before the relation is used.
       // If we can't find it, it's a bug.
       throw new ReplicationAssertionError(`Missing relation cache for ${tableId}`);
     }
-    return table;
+    return tables;
   }
 
   async streamChanges() {
-    const serverId = createRandomServerId(this.storage.group_id);
+    const serverId = createRandomServerId(this.storage.replicationStreamId);
 
     const connection = await this.connections.getConnection();
-    const { checkpoint_lsn } = await this.storage.getStatus();
-    if (checkpoint_lsn) {
-      this.logger.info(`Existing checkpoint found: ${checkpoint_lsn}`);
+    const { resumeLsn: resume_lsn } = await this.storage.getStatus();
+    if (resume_lsn) {
+      this.logger.info(`Existing resume LSN found: ${resume_lsn}`);
     }
-    const fromGTID = checkpoint_lsn
-      ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn)
+    const fromGTID = resume_lsn
+      ? common.ReplicatedGTID.fromSerialized(resume_lsn)
       : await common.readExecutedGtid(connection);
     connection.release();
 
@@ -498,10 +503,10 @@ export class BinLogStream {
     if (change.type === SchemaChangeType.RENAME_TABLE) {
       const fromTableId = createTableId(change.schema, change.table);
 
-      const fromTable = this.tableCache.get(fromTableId);
+      const fromTables = this.tableCache.get(fromTableId);
       // Old table needs to be cleaned up
-      if (fromTable) {
-        await batch.drop([fromTable]);
+      if (fromTables) {
+        await batch.drop(fromTables);
         this.tableCache.delete(fromTableId);
       }
       // The new table matched a table in the sync config
@@ -511,7 +516,7 @@ export class BinLogStream {
     } else {
       const tableId = createTableId(change.schema, change.table);
 
-      const table = this.getTable(tableId);
+      const tables = this.getTables(tableId);
 
       switch (change.type) {
         case SchemaChangeType.ALTER_TABLE_COLUMN:
@@ -520,10 +525,10 @@ export class BinLogStream {
           await this.handleCreateOrUpdateTable(batch, change.table, change.schema);
           break;
         case SchemaChangeType.TRUNCATE_TABLE:
-          await batch.truncate([table]);
+          await batch.truncate(tables);
           break;
         case SchemaChangeType.DROP_TABLE:
-          await batch.drop([table]);
+          await batch.drop(tables);
           this.tableCache.delete(tableId);
           break;
         default:
@@ -549,13 +554,14 @@ export class BinLogStream {
     batch: storage.BucketStorageBatch,
     tableName: string,
     schema: string
-  ): Promise<SourceTable> {
+  ): Promise<SourceTable[]> {
     const replicaIdColumns = await this.getReplicaIdColumns(tableName, schema);
     return await this.handleRelation(
       batch,
       {
         name: tableName,
         schema: schema,
+        connectionTag: this.connectionTag,
         objectId: createTableId(schema, tableName),
         replicaIdColumns: replicaIdColumns
       },
@@ -575,23 +581,25 @@ export class BinLogStream {
     const columns = common.toColumnDescriptors(msg.tableEntry);
     const tableId = createTableId(msg.tableEntry.parentSchema, msg.tableEntry.tableName);
 
-    let table = this.tableCache.get(tableId);
-    if (table == null) {
+    let tables = this.tableCache.get(tableId);
+    if (tables == null) {
       // This is an insert for a new table that matches a table in the sync config
       // We need to create the table in the storage and cache it.
-      table = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema);
+      tables = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema);
     }
 
     for (const [index, row] of msg.rows.entries()) {
-      await this.writeChange(batch, {
-        type: msg.type,
-        database: msg.tableEntry.parentSchema,
-        sourceTable: table!,
-        table: msg.tableEntry.tableName,
-        columns: columns,
-        row: row,
-        previous_row: msg.rows_before?.[index]
-      });
+      for (const table of tables.filter((table) => table.syncAny)) {
+        await this.writeChange(batch, {
+          type: msg.type,
+          database: msg.tableEntry.parentSchema,
+          sourceTable: table,
+          table: msg.tableEntry.tableName,
+          columns: columns,
+          row: row,
+          previous_row: msg.rows_before?.[index]
+        });
+      }
     }
     return null;
   }

package/src/utils/mysql-utils.ts

@@ -1,5 +1,5 @@
 import { logger } from '@powersync/lib-services-framework';
-import { SourceEntityDescriptor } from '@powersync/service-core';
+import { SourceTableRef } from '@powersync/service-sync-rules';
 import mysql from 'mysql2';
 import mysqlPromise from 'mysql2/promise';
 import { coerce, gte, satisfies } from 'semver';
@@ -101,10 +101,10 @@ export function satisfiesVersion(version: string, targetVersion: string): boolea
   return satisfies(coercedVersion!, targetVersion!, { loose: true });
 }
 
-export function qualifiedMySQLTable(table: SourceEntityDescriptor): string;
+export function qualifiedMySQLTable(table: SourceTableRef): string;
 export function qualifiedMySQLTable(table: string, schema: string): string;
 
-export function qualifiedMySQLTable(table: SourceEntityDescriptor | string, schema?: string): string {
+export function qualifiedMySQLTable(table: SourceTableRef | string, schema?: string): string {
   if (typeof table === 'object') {
     return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``;
   } else if (schema) {

package/test/src/BinlogStreamUtils.ts

@@ -33,7 +33,7 @@ export class BinlogStreamTestContext {
   private streamPromise?: Promise<void>;
   public storage?: SyncRulesBucketStorage;
   private replicationDone = false;
-  private syncRulesContent?: storage.PersistedSyncRulesContent;
+  private syncRulesContent?: storage.PersistedSyncConfigContent;
 
   static async open(factory: storage.TestStorageFactory, options?: { doNotClear?: boolean }) {
     const f = await factory({ doNotClear: options?.doNotClear });
@@ -71,38 +71,38 @@ export class BinlogStreamTestContext {
   }
 
   async updateSyncRules(content: string): Promise<SyncRulesBucketStorage> {
-    const syncRules = await this.factory.updateSyncRules(
+    const replicationStream = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION })
     );
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = replicationStream.syncConfigContent[0];
+    this.storage = this.factory.getInstance(replicationStream);
     return this.storage!;
   }
 
   async loadNextSyncRules() {
-    const syncRules = await this.factory.getNextSyncRulesContent();
-    if (syncRules == null) {
+    const syncConfig = await this.factory.getDeployingSyncConfig();
+    if (syncConfig == null) {
       throw new Error(`Next replication stream not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
   async loadActiveSyncRules() {
-    const syncRules = await this.factory.getActiveSyncRulesContent();
-    if (syncRules == null) {
+    const syncConfig = await this.factory.getActiveSyncConfig();
+    if (syncConfig == null) {
       throw new Error(`Active replication stream not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.storage = syncConfig.storage;
     this.replicationDone = true;
     return this.storage!;
   }
 
-  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+  private getSyncConfigContent(): storage.PersistedSyncConfigContent {
     if (this.syncRulesContent == null) {
       throw new Error('Sync config not configured - call updateSyncRules() first');
     }
@@ -165,8 +165,8 @@ export class BinlogStreamTestContext {
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     const checkpoint = await this.getCheckpoint(options);
-    const syncRules = this.getSyncRulesContent();
-    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
+    const syncConfigContent = this.getSyncConfigContent();
+    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncConfigContent, bucket, start));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -179,9 +179,9 @@ export class BinlogStreamTestContext {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
-    const syncRules = this.getSyncRulesContent();
+    const syncConfigContent = this.getSyncConfigContent();
     const checkpoint = await this.getCheckpoint(options);
-    const map = [bucketRequest(syncRules, bucket, start)];
+    const map = [bucketRequest(syncConfigContent, bucket, start)];
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await test_utils.fromAsync(batch);
     return batches[0]?.chunkData.data ?? [];
@@ -203,7 +203,7 @@ export async function getClientCheckpoint(
 
   logger.info('Expected Checkpoint: ' + gtid.comparable);
   while (Date.now() - start < timeout) {
-    const storage = await storageFactory.getActiveStorage();
+    const storage = (await storageFactory.getActiveSyncConfig())?.storage;
     const cp = await storage?.getCheckpoint();
     if (cp == null) {
       throw new Error('No replication stream available');

package/tsconfig.json

@@ -3,9 +3,6 @@
   "compilerOptions": {
     "rootDir": "src",
     "outDir": "dist",
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "sourceMap": true,
     "typeRoots": ["./node_modules/@types", "./src/replication/zongji.d.ts"]
   },
   "include": ["src"],