@powersync/service-module-mysql 0.7.4 → 0.8.0

package/dist/utils/mysql-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,MAAM,QAAQ,CAAC;AAG3B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAUrC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,KAAK,IAAI,KAAK,GAAG,OAAO,GAAI,KAAK,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;YAC1C,OAAO,UAAU,CAAC,KAAK,CAA+B,KAAK,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA6C,EAAE,OAA2B;IACnG,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,GAAG,EAAE,MAAM,CAAC,kBAAkB;QAC9B,IAAI,EAAE,MAAM,CAAC,kBAAkB;KAChC,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,6CAA6C;IAC7C,OAAO,KAAK,CAAC,UAAU,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC3C,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,GAAG,EAAE,0DAA0D;QACzE,WAAW,EAAE,IAAI,EAAE,iCAAiC;QACpD,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAAmC;IACvE,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,MAAM,YAAY,CAAC;QAC3C,UAAU;QACV,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,OAAiB,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,cAAsB;IACtE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,qBAAqB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAErD,OAAO,GAAG,CAAC,cAAe,EAAE,qBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;AAC9F,CAAC"}
+{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,MAAM,QAAQ,CAAC;AAG3B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAUhD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,KAAK,IAAI,KAAK,GAAG,OAAO,GAAI,KAAK,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;YAC1C,OAAO,UAAU,CAAC,KAAK,CAA+B,KAAK,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA6C,EAAE,OAA2B;IACnG,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,GAAG,EAAE,MAAM,CAAC,kBAAkB;QAC9B,IAAI,EAAE,MAAM,CAAC,kBAAkB;KAChC,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,6CAA6C;IAC7C,OAAO,KAAK,CAAC,UAAU,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC3C,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,GAAG,EAAE,0DAA0D;QACzE,WAAW,EAAE,IAAI,EAAE,iCAAiC;QACpD,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAAmC;IACvE,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,MAAM,YAAY,CAAC;QAC3C,UAAU;QACV,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,OAAiB,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,cAAsB;IACtE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,qBAAqB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAErD,OAAO,GAAG,CAAC,cAAe,EAAE,qBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,aAAqB;IACrE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAEvC,OAAO,SAAS,CAAC,cAAe,EAAE,aAAc,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACrE,CAAC;AAKD,MAAM,UAAU,mBAAmB,CAAC,KAAsC,EAAE,MAAe;IACzF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAC7F,CAAC;SAAM,IAAI,MAAM,EAAE,CAAC;QAClB,OAAO,KAAK,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAClF,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAC9C,CAAC;AACH,CAAC"}

package/dist/utils/parser-utils.d.ts

@@ -0,0 +1,16 @@
+import { Alter, AST, Create, Drop, TruncateStatement, RenameStatement, DropIndexStatement } from 'node-sql-parser';
+/**
+ *  Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions.
+ *  @param query
+ *  @param matchers
+ */
+export declare function matchedSchemaChangeQuery(query: string, matchers: ((table: string) => boolean)[]): boolean;
+export declare function isTruncate(statement: AST): statement is TruncateStatement;
+export declare function isRenameTable(statement: AST): statement is RenameStatement;
+export declare function isAlterTable(statement: AST): statement is Alter;
+export declare function isRenameExpression(expression: any): boolean;
+export declare function isColumnExpression(expression: any): boolean;
+export declare function isConstraintExpression(expression: any): boolean;
+export declare function isDropTable(statement: AST): statement is Drop;
+export declare function isDropIndex(statement: AST): statement is DropIndexStatement;
+export declare function isCreateUniqueIndex(statement: AST): statement is Create;

package/dist/utils/parser-utils.js

@@ -0,0 +1,58 @@
+// We ignore create table statements, since even in the worst case we will pick up the changes when row events for that
+// table are received.
+const DDL_KEYWORDS = ['alter table', 'drop table', 'truncate table', 'rename table'];
+/**
+ *  Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions.
+ *  @param query
+ *  @param matchers
+ */
+export function matchedSchemaChangeQuery(query, matchers) {
+    // Normalize case and remove backticks for matching
+    const normalizedQuery = query.toLowerCase().replace(/`/g, '');
+    const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword));
+    if (isDDLQuery) {
+        const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/);
+        // Check if any matched table names appear in the query
+        for (const token of tokens) {
+            const matchFound = matchers.some((matcher) => matcher(token));
+            if (matchFound) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+// @ts-ignore
+export function isTruncate(statement) {
+    // @ts-ignore
+    return statement.type === 'truncate';
+}
+// @ts-ignore
+export function isRenameTable(statement) {
+    // @ts-ignore
+    return statement.type === 'rename';
+}
+export function isAlterTable(statement) {
+    return statement.type === 'alter';
+}
+export function isRenameExpression(expression) {
+    return expression.resource === 'table' && expression.action === 'rename';
+}
+export function isColumnExpression(expression) {
+    return expression.resource === 'column';
+}
+export function isConstraintExpression(expression) {
+    return ((expression.resource === 'key' && expression.keyword === 'primary key') ||
+        expression.resource === 'constraint' ||
+        (expression.resource === 'index' && expression.action === 'drop'));
+}
+export function isDropTable(statement) {
+    return statement.type === 'drop' && statement.keyword === 'table';
+}
+export function isDropIndex(statement) {
+    return statement.type === 'drop' && statement.keyword === 'index';
+}
+export function isCreateUniqueIndex(statement) {
+    return statement.type === 'create' && statement.keyword === 'index' && statement.index_type === 'unique';
+}
+//# sourceMappingURL=parser-utils.js.map

package/dist/utils/parser-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser-utils.js","sourceRoot":"","sources":["../../src/utils/parser-utils.ts"],"names":[],"mappings":"AAEA,uHAAuH;AACvH,sBAAsB;AACtB,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;AAErF;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAa,EAAE,QAAwC;IAC9F,mDAAmD;IACnD,MAAM,eAAe,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACxD,uDAAuD;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9D,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,aAAa;AACb,MAAM,UAAU,UAAU,CAAC,SAAc;IACvC,aAAa;IACb,OAAO,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC;AACvC,CAAC;AAED,aAAa;AACb,MAAM,UAAU,aAAa,CAAC,SAAc;IAC1C,aAAa;IACb,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAc;IACzC,OAAO,SAAS,CAAC,IAAI,KAAK,OAAO,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,UAAe;IAChD,OAAO,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,QAAQ,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,UAAe;IAChD,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAe;IACpD,OAAO,CACL,CAAC,UAAU,CAAC,QAAQ,KAAK,KAAK,IAAI,UAAU,CAAC,OAAO,KAAK,aAAa,CAAC;QACvE,UAAU,CAAC,QAAQ,KAAK,YAAY;QACpC,CAAC,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,CAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAc;IACxC,OAAO,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,OAAO,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,SAAc;IACxC,OAAO,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,OAAO,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,SAAc;IAChD,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,OAAO,KAAK,OAAO,IAAI,SAAS,CAAC,UAAU,KAAK,QAAQ,CAAC;AAC3G,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mysql",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.7.4",
+  "version": "0.8.0",
   "license": "FSL-1.1-Apache-2.0",
   "main": "dist/index.js",
   "type": "module",
@@ -22,25 +22,26 @@
     }
   },
   "dependencies": {
-    "@powersync/mysql-zongji": "0.2.0",
+    "@powersync/mysql-zongji": "^0.4.0",
     "async": "^3.2.4",
     "mysql2": "^3.11.0",
+    "node-sql-parser": "^5.3.9",
     "semver": "^7.5.4",
     "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^11.1.0",
-    "@powersync/lib-services-framework": "0.7.1",
-    "@powersync/service-core": "1.13.4",
-    "@powersync/service-sync-rules": "0.27.0",
+    "@powersync/lib-services-framework": "0.7.2",
+    "@powersync/service-core": "1.14.0",
+    "@powersync/service-sync-rules": "0.28.0",
     "@powersync/service-types": "0.12.1",
     "@powersync/service-jsonbig": "0.17.10"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/semver": "^7.5.4",
-    "@powersync/service-core-tests": "0.10.4",
-    "@powersync/service-module-mongodb-storage": "0.10.4",
-    "@powersync/service-module-postgres-storage": "0.8.4"
+    "@powersync/service-core-tests": "0.11.0",
+    "@powersync/service-module-mongodb-storage": "0.11.0",
+    "@powersync/service-module-postgres-storage": "0.9.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/MySQLRouteAPIAdapter.ts

@@ -208,7 +208,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
       idColumnsResult = await common.getReplicationIdentityColumns({
         connection: connection,
         schema,
-        table_name: tableName
+        tableName: tableName
       });
     } catch (ex) {
       idColumnsError = { level: 'fatal', message: ex.message };
@@ -217,7 +217,15 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     }
 
     const idColumns = idColumnsResult?.columns ?? [];
-    const sourceTable = new storage.SourceTable(0, this.config.tag, tableName, schema, tableName, idColumns, true);
+    const sourceTable = new storage.SourceTable({
+      id: 0,
+      connectionTag: this.config.tag,
+      objectId: tableName,
+      schema: schema,
+      name: tableName,
+      replicaIdColumns: idColumns,
+      snapshotComplete: true
+    });
     const syncData = syncRules.tableSyncsData(sourceTable);
     const syncParameters = syncRules.tableSyncsParameters(sourceTable);
 
@@ -232,7 +240,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     let selectError: service_types.ReplicationError | null = null;
     try {
       await this.retriedQuery({
-        query: `SELECT * FROM ${sourceTable.table} LIMIT 1`
+        query: `SELECT * FROM ${sourceTable.name} LIMIT 1`
       });
     } catch (e) {
       selectError = { level: 'fatal', message: e.message };

package/src/common/ReplicatedGTID.ts

@@ -92,10 +92,15 @@ export class ReplicatedGTID {
    * @returns A comparable string in the format
    *   `padded_end_transaction|raw_gtid|binlog_filename|binlog_position`
    */
-  get comparable() {
+  get comparable(): string {
     const { raw, position } = this;
     const [, transactionRanges] = this.raw.split(':');
 
+    // This means no transactions have been executed on the database yet
+    if (!transactionRanges) {
+      return ReplicatedGTID.ZERO.comparable;
+    }
+
     let maxTransactionId = 0;
 
     for (const range of transactionRanges.split(',')) {

package/src/common/common-index.ts

@@ -1,6 +1,5 @@
 export * from './check-source-configuration.js';
-export * from './get-replication-columns.js';
-export * from './get-tables-from-pattern.js';
+export * from './schema-utils.js';
 export * from './mysql-to-sqlite.js';
 export * from './read-executed-gtid.js';
 export * from './ReplicatedGTID.js';

package/src/common/mysql-to-sqlite.ts

@@ -183,6 +183,9 @@ export function toSQLiteRow(row: Record<string, any>, columns: Map<string, Colum
           result[key] = row[key];
           break;
       }
+    } else {
+      // If the value is null, we just set it to null
+      result[key] = null;
     }
   }
   return sync_rules.toSyncRulesRow(result);

package/src/common/{get-replication-columns.ts → schema-utils.ts}

@@ -1,23 +1,64 @@
-import { storage } from '@powersync/service-core';
 import mysqlPromise from 'mysql2/promise';
 import * as mysql_utils from '../utils/mysql-utils.js';
+import { ColumnDescriptor } from '@powersync/service-core';
+import { TablePattern } from '@powersync/service-sync-rules';
 
-export type GetReplicationColumnsOptions = {
+export interface GetColumnsOptions {
   connection: mysqlPromise.Connection;
   schema: string;
-  table_name: string;
-};
+  tableName: string;
+}
+
+export async function getColumns(options: GetColumnsOptions): Promise<ColumnDescriptor[]> {
+  const { connection, schema, tableName } = options;
+
+  const [allColumns] = await mysql_utils.retriedQuery({
+    connection: connection,
+    query: `
+      SELECT 
+        s.COLUMN_NAME AS name,
+        c.DATA_TYPE as type
+      FROM 
+        INFORMATION_SCHEMA.COLUMNS s
+        JOIN 
+          INFORMATION_SCHEMA.COLUMNS c
+            ON 
+              s.TABLE_SCHEMA = c.TABLE_SCHEMA
+              AND s.TABLE_NAME = c.TABLE_NAME
+              AND s.COLUMN_NAME = c.COLUMN_NAME
+      WHERE 
+        s.TABLE_SCHEMA = ?
+        AND s.TABLE_NAME = ?
+      ORDER BY 
+        s.ORDINAL_POSITION;
+      `,
+    params: [schema, tableName]
+  });
+
+  return allColumns.map((row) => {
+    return {
+      name: row.name,
+      type: row.type
+    };
+  });
+}
+
+export interface GetReplicationIdentityColumnsOptions {
+  connection: mysqlPromise.Connection;
+  schema: string;
+  tableName: string;
+}
 
-export type ReplicationIdentityColumnsResult = {
-  columns: storage.ColumnDescriptor[];
+export interface ReplicationIdentityColumnsResult {
+  columns: ColumnDescriptor[];
   //   TODO maybe export an enum from the core package
   identity: string;
-};
+}
 
 export async function getReplicationIdentityColumns(
-  options: GetReplicationColumnsOptions
+  options: GetReplicationIdentityColumnsOptions
 ): Promise<ReplicationIdentityColumnsResult> {
-  const { connection, schema, table_name } = options;
+  const { connection, schema, tableName } = options;
   const [primaryKeyColumns] = await mysql_utils.retriedQuery({
     connection: connection,
     query: `
@@ -39,7 +80,7 @@ export async function getReplicationIdentityColumns(
       ORDER BY 
         s.SEQ_IN_INDEX;
       `,
-    params: [schema, table_name]
+    params: [schema, tableName]
   });
 
   if (primaryKeyColumns.length) {
@@ -52,8 +93,7 @@ export async function getReplicationIdentityColumns(
     };
   }
 
-  // TODO: test code with tables with unique keys, compound key etc.
-  // No primary key, find the first valid unique key
+  // No primary key, check if any of the columns have a unique constraint we can use
   const [uniqueKeyColumns] = await mysql_utils.retriedQuery({
     connection: connection,
     query: `
@@ -78,7 +118,7 @@ export async function getReplicationIdentityColumns(
         AND s.NON_UNIQUE = 0
       ORDER BY s.SEQ_IN_INDEX;
       `,
-    params: [schema, table_name]
+    params: [schema, tableName]
   });
 
   if (uniqueKeyColumns.length > 0) {
@@ -91,34 +131,53 @@ export async function getReplicationIdentityColumns(
     };
   }
 
-  const [allColumns] = await mysql_utils.retriedQuery({
+  const allColumns = await getColumns({
     connection: connection,
-    query: `
-      SELECT 
-        s.COLUMN_NAME AS name,
-        c.DATA_TYPE as type
-      FROM 
-        INFORMATION_SCHEMA.COLUMNS s
-        JOIN 
-          INFORMATION_SCHEMA.COLUMNS c
-            ON 
-              s.TABLE_SCHEMA = c.TABLE_SCHEMA
-              AND s.TABLE_NAME = c.TABLE_NAME
-              AND s.COLUMN_NAME = c.COLUMN_NAME
-      WHERE 
-        s.TABLE_SCHEMA = ?
-        AND s.TABLE_NAME = ?
-      ORDER BY 
-        s.ORDINAL_POSITION;
-      `,
-    params: [schema, table_name]
+    schema: schema,
+    tableName: tableName
   });
 
   return {
-    columns: allColumns.map((row) => ({
-      name: row.name,
-      type: row.type
-    })),
+    columns: allColumns,
     identity: 'full'
   };
 }
+
+export async function getTablesFromPattern(
+  connection: mysqlPromise.Connection,
+  tablePattern: TablePattern
+): Promise<string[]> {
+  const schema = tablePattern.schema;
+
+  if (tablePattern.isWildcard) {
+    const [results] = await mysql_utils.retriedQuery({
+      connection: connection,
+      query: `
+          SELECT TABLE_NAME
+          FROM information_schema.tables
+          WHERE TABLE_SCHEMA = ?
+            AND TABLE_NAME LIKE ?
+            AND table_type = 'BASE TABLE'
+      `,
+      params: [schema, tablePattern.tablePattern]
+    });
+
+    return results
+      .map((row) => row.TABLE_NAME)
+      .filter((tableName: string) => tableName.startsWith(tablePattern.tablePrefix));
+  } else {
+    const [results] = await mysql_utils.retriedQuery({
+      connection: connection,
+      query: `
+          SELECT TABLE_NAME
+          FROM information_schema.tables
+          WHERE TABLE_SCHEMA = ?
+            AND TABLE_NAME = ?
+            AND table_type = 'BASE TABLE'
+        `,
+      params: [schema, tablePattern.tablePattern]
+    });
+
+    return results.map((row) => row.TABLE_NAME);
+  }
+}