@powersync/service-module-mysql 0.12.3 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/api/MySQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/common-index.d.ts +1 -1
- package/dist/common/common-index.js +1 -1
- package/dist/common/common-index.js.map +1 -1
- package/dist/common/mysql-to-sqlite.d.ts +2 -2
- package/dist/common/mysql-to-sqlite.js +1 -1
- package/dist/common/mysql-to-sqlite.js.map +1 -1
- package/dist/common/schema-utils.d.ts +1 -1
- package/dist/common/schema-utils.js.map +1 -1
- package/dist/module/MySQLModule.js +3 -3
- package/dist/module/MySQLModule.js.map +1 -1
- package/dist/replication/BinLogReplicationJob.js +2 -2
- package/dist/replication/BinLogReplicationJob.js.map +1 -1
- package/dist/replication/BinLogReplicator.js +1 -1
- package/dist/replication/BinLogReplicator.js.map +1 -1
- package/dist/replication/BinLogStream.js +4 -4
- package/dist/replication/BinLogStream.js.map +1 -1
- package/dist/replication/MySQLConnectionManager.d.ts +3 -3
- package/dist/replication/MySQLConnectionManager.js +1 -1
- package/dist/replication/MySQLConnectionManager.js.map +1 -1
- package/dist/replication/MySQLConnectionManagerFactory.d.ts +1 -1
- package/dist/replication/MySQLConnectionManagerFactory.js.map +1 -1
- package/dist/replication/zongji/BinLogListener.d.ts +4 -4
- package/dist/replication/zongji/BinLogListener.js +7 -7
- package/dist/replication/zongji/BinLogListener.js.map +1 -1
- package/dist/replication/zongji/zongji-utils.d.ts +1 -1
- package/dist/utils/mysql-utils.d.ts +2 -2
- package/dist/utils/mysql-utils.js +1 -1
- package/dist/utils/mysql-utils.js.map +1 -1
- package/dist/utils/parser-utils.d.ts +1 -1
- package/package.json +10 -10
- package/src/api/MySQLRouteAPIAdapter.ts +1 -1
- package/src/common/common-index.ts +1 -1
- package/src/common/mysql-to-sqlite.ts +3 -3
- package/src/common/schema-utils.ts +2 -2
- package/src/module/MySQLModule.ts +3 -3
- package/src/replication/BinLogReplicationJob.ts +2 -2
- package/src/replication/BinLogReplicator.ts +1 -1
- package/src/replication/BinLogStream.ts +4 -4
- package/src/replication/MySQLConnectionManager.ts +4 -4
- package/src/replication/MySQLConnectionManagerFactory.ts +1 -1
- package/src/replication/zongji/BinLogListener.ts +10 -10
- package/src/replication/zongji/zongji-utils.ts +6 -6
- package/src/utils/mysql-utils.ts +3 -3
- package/src/utils/parser-utils.ts +1 -1
- package/test/src/BinLogListener.test.ts +11 -7
- package/test/src/BinLogStream.test.ts +1 -1
- package/test/src/BinlogStreamUtils.ts +5 -5
- package/test/src/config.test.ts +1 -1
- package/test/src/mysql-to-sqlite.test.ts +5 -6
- package/test/src/mysql-utils.test.ts +1 -1
- package/test/src/parser-utils.test.ts +1 -1
- package/test/src/schema-changes.test.ts +4 -4
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { SourceEntityDescriptor } from '@powersync/service-core';
|
|
1
2
|
import mysql from 'mysql2';
|
|
2
3
|
import mysqlPromise from 'mysql2/promise';
|
|
3
4
|
import * as types from '../types/types.js';
|
|
4
|
-
import { SourceEntityDescriptor } from '@powersync/service-core';
|
|
5
5
|
export type RetriedQueryOptions = {
|
|
6
6
|
connection: mysqlPromise.Connection;
|
|
7
7
|
query: string;
|
|
@@ -14,7 +14,7 @@ export type RetriedQueryOptions = {
|
|
|
14
14
|
export declare function retriedQuery(options: RetriedQueryOptions): Promise<[mysql.RowDataPacket[], mysql.FieldPacket[]]>;
|
|
15
15
|
export declare function createPool(config: types.NormalizedMySQLConnectionConfig, options?: mysql.PoolOptions): mysql.Pool;
|
|
16
16
|
/**
|
|
17
|
-
* Return a random server id for a given
|
|
17
|
+
* Return a random server id for a given replication stream id.
|
|
18
18
|
* Expected format is: <syncRuleId>00<random number>
|
|
19
19
|
* The max value for server id in MySQL is 2^32 - 1.
|
|
20
20
|
* We use the GTID format to keep track of our position in the binlog, no state is kept by the MySQL server, therefore
|
|
@@ -49,7 +49,7 @@ export function createPool(config, options) {
|
|
|
49
49
|
});
|
|
50
50
|
}
|
|
51
51
|
/**
|
|
52
|
-
* Return a random server id for a given
|
|
52
|
+
* Return a random server id for a given replication stream id.
|
|
53
53
|
* Expected format is: <syncRuleId>00<random number>
|
|
54
54
|
* The max value for server id in MySQL is 2^32 - 1.
|
|
55
55
|
* We use the GTID format to keep track of our position in the binlog, no state is kept by the MySQL server, therefore
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,MAAM,QAAQ,CAAC;AAE3B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAUhD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,KAAK,IAAI,KAAK,GAAG,OAAO,GAAI,KAAK,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;YAC1C,OAAO,UAAU,CAAC,KAAK,CAA+B,KAAK,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA6C,EAAE,OAA2B;IACnG,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,GAAG,EAAE,MAAM,CAAC,kBAAkB;QAC9B,IAAI,EAAE,MAAM,CAAC,kBAAkB;KAChC,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjE,+EAA+E;IAC/E,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAEvC,6CAA6C;IAC7C,OAAO,KAAK,CAAC,UAAU,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC3C,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,GAAG,EAAE,0DAA0D;QACzE,WAAW,EAAE,IAAI,EAAE,iCAAiC;QACpD,WAAW,EAAE,IAAI,EAAE,qCAAqC;QACxD,qFAAqF;QACrF,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAAmC;IACvE,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,MAAM,YAAY,CAAC;QAC3C,UAAU;QACV,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,OAAiB,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,cAAsB;IACtE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,qBAAqB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAErD,OAAO,GAAG,CAAC,cAAe,EAAE,qBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,aAAqB;IACrE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAEvC,OAAO,SAAS,CAAC,cAAe,EAAE,aAAc,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACrE,CAAC;AAKD,MAAM,UAAU,mBAAmB,CAAC,KAAsC,EAAE,MAAe;IACzF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAC7F,CAAC;SAAM,IAAI,MAAM,EAAE,CAAC;QAClB,OAAO,KAAK,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAClF,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;IAC9C,CAAC;AACH,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Alter, AST, Create, Drop,
|
|
1
|
+
import { Alter, AST, Create, Drop, DropIndexStatement, RenameStatement, TruncateStatement } from 'node-sql-parser';
|
|
2
2
|
/**
|
|
3
3
|
* Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions.
|
|
4
4
|
* @param query
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mysql",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.12.
|
|
5
|
+
"version": "0.12.5",
|
|
6
6
|
"license": "FSL-1.1-ALv2",
|
|
7
7
|
"main": "dist/index.js",
|
|
8
8
|
"type": "module",
|
|
@@ -29,19 +29,19 @@
|
|
|
29
29
|
"semver": "^7.5.4",
|
|
30
30
|
"ts-codec": "^1.3.0",
|
|
31
31
|
"uri-js": "^4.4.1",
|
|
32
|
-
"uuid": "^
|
|
33
|
-
"@powersync/lib-services-framework": "0.9.
|
|
34
|
-
"@powersync/service-core": "1.
|
|
35
|
-
"@powersync/service-sync-rules": "0.
|
|
36
|
-
"@powersync/service-types": "0.15.
|
|
37
|
-
"@powersync/service-jsonbig": "0.17.
|
|
32
|
+
"uuid": "^14.0.0",
|
|
33
|
+
"@powersync/lib-services-framework": "0.9.4",
|
|
34
|
+
"@powersync/service-core": "1.21.0",
|
|
35
|
+
"@powersync/service-sync-rules": "0.36.0",
|
|
36
|
+
"@powersync/service-types": "0.15.2",
|
|
37
|
+
"@powersync/service-jsonbig": "0.17.13"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@types/async": "^3.2.24",
|
|
41
41
|
"@types/semver": "^7.7.1",
|
|
42
|
-
"@powersync/service-core-tests": "0.15.
|
|
43
|
-
"@powersync/service-module-mongodb-storage": "0.
|
|
44
|
-
"@powersync/service-module-postgres-storage": "0.
|
|
42
|
+
"@powersync/service-core-tests": "0.15.5",
|
|
43
|
+
"@powersync/service-module-mongodb-storage": "0.16.0",
|
|
44
|
+
"@powersync/service-module-postgres-storage": "0.14.0"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc -b",
|
|
@@ -5,8 +5,8 @@ import * as service_types from '@powersync/service-types';
|
|
|
5
5
|
import mysql from 'mysql2/promise';
|
|
6
6
|
import * as common from '../common/common-index.js';
|
|
7
7
|
import { toExpressionTypeFromMySQLType } from '../common/common-index.js';
|
|
8
|
-
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
9
8
|
import * as types from '../types/types.js';
|
|
9
|
+
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
10
10
|
|
|
11
11
|
type SchemaResult = {
|
|
12
12
|
schema_name: string;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import { ColumnDefinition, TableMapEntry } from '@powersync/mysql-zongji';
|
|
2
|
+
import { ColumnDescriptor } from '@powersync/service-core';
|
|
3
|
+
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
1
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
2
5
|
import { ExpressionType } from '@powersync/service-sync-rules';
|
|
3
|
-
import { ColumnDescriptor } from '@powersync/service-core';
|
|
4
6
|
import mysql from 'mysql2';
|
|
5
|
-
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
6
|
-
import { ColumnDefinition, TableMapEntry } from '@powersync/mysql-zongji';
|
|
7
7
|
|
|
8
8
|
export enum ADDITIONAL_MYSQL_TYPES {
|
|
9
9
|
DATETIME2 = 18,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import mysqlPromise from 'mysql2/promise';
|
|
2
|
-
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
3
1
|
import { ColumnDescriptor } from '@powersync/service-core';
|
|
4
2
|
import { TablePattern } from '@powersync/service-sync-rules';
|
|
3
|
+
import mysqlPromise from 'mysql2/promise';
|
|
4
|
+
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
5
5
|
|
|
6
6
|
export interface GetColumnsOptions {
|
|
7
7
|
connection: mysqlPromise.Connection;
|
|
@@ -8,13 +8,13 @@ import {
|
|
|
8
8
|
} from '@powersync/service-core';
|
|
9
9
|
|
|
10
10
|
import { MySQLRouteAPIAdapter } from '../api/MySQLRouteAPIAdapter.js';
|
|
11
|
+
import { checkSourceConfiguration } from '../common/check-source-configuration.js';
|
|
11
12
|
import { BinLogReplicator } from '../replication/BinLogReplicator.js';
|
|
13
|
+
import { MySQLConnectionManager } from '../replication/MySQLConnectionManager.js';
|
|
14
|
+
import { MySQLConnectionManagerFactory } from '../replication/MySQLConnectionManagerFactory.js';
|
|
12
15
|
import { MySQLErrorRateLimiter } from '../replication/MySQLErrorRateLimiter.js';
|
|
13
16
|
import * as types from '../types/types.js';
|
|
14
|
-
import { MySQLConnectionManagerFactory } from '../replication/MySQLConnectionManagerFactory.js';
|
|
15
17
|
import { MySQLConnectionConfig } from '../types/types.js';
|
|
16
|
-
import { checkSourceConfiguration } from '../common/check-source-configuration.js';
|
|
17
|
-
import { MySQLConnectionManager } from '../replication/MySQLConnectionManager.js';
|
|
18
18
|
|
|
19
19
|
export class MySQLModule extends replication.ReplicationModule<types.MySQLConnectionConfig> {
|
|
20
20
|
constructor() {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { container
|
|
1
|
+
import { container } from '@powersync/lib-services-framework';
|
|
2
2
|
import { POWERSYNC_VERSION, replication } from '@powersync/service-core';
|
|
3
3
|
import { BinLogStream } from './BinLogStream.js';
|
|
4
4
|
import { MySQLConnectionManagerFactory } from './MySQLConnectionManagerFactory.js';
|
|
@@ -13,7 +13,7 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
|
|
|
13
13
|
|
|
14
14
|
constructor(options: BinLogReplicationJobOptions) {
|
|
15
15
|
super(options);
|
|
16
|
-
this.logger =
|
|
16
|
+
this.logger = options.storage.logger;
|
|
17
17
|
this.connectionFactory = options.connectionFactory;
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { replication, storage } from '@powersync/service-core';
|
|
2
|
+
import { MySQLModule } from '../module/MySQLModule.js';
|
|
2
3
|
import { BinLogReplicationJob } from './BinLogReplicationJob.js';
|
|
3
4
|
import { MySQLConnectionManagerFactory } from './MySQLConnectionManagerFactory.js';
|
|
4
|
-
import { MySQLModule } from '../module/MySQLModule.js';
|
|
5
5
|
|
|
6
6
|
export interface BinLogReplicatorOptions extends replication.AbstractReplicatorOptions {
|
|
7
7
|
connectionFactory: MySQLConnectionManagerFactory;
|
|
@@ -139,7 +139,7 @@ export class BinLogStream {
|
|
|
139
139
|
// Snapshot if:
|
|
140
140
|
// 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
|
|
141
141
|
// 2. Snapshot is not done yet, AND:
|
|
142
|
-
// 3. The table is used in sync
|
|
142
|
+
// 3. The table is used in sync config.
|
|
143
143
|
const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny;
|
|
144
144
|
|
|
145
145
|
if (shouldSnapshot) {
|
|
@@ -291,7 +291,7 @@ export class BinLogStream {
|
|
|
291
291
|
}
|
|
292
292
|
|
|
293
293
|
if (lastOp != null) {
|
|
294
|
-
// Populate the cache _after_ initial replication, but _before_ we switch to this
|
|
294
|
+
// Populate the cache _after_ initial replication, but _before_ we switch to this replication stream.
|
|
295
295
|
await this.storage.populatePersistentChecksumCache({
|
|
296
296
|
// No checkpoint yet, but we do have the opId.
|
|
297
297
|
maxOpId: lastOp,
|
|
@@ -504,7 +504,7 @@ export class BinLogStream {
|
|
|
504
504
|
await batch.drop([fromTable]);
|
|
505
505
|
this.tableCache.delete(fromTableId);
|
|
506
506
|
}
|
|
507
|
-
// The new table matched a table in the sync
|
|
507
|
+
// The new table matched a table in the sync config
|
|
508
508
|
if (change.newTable) {
|
|
509
509
|
await this.handleCreateOrUpdateTable(batch, change.newTable!, change.schema);
|
|
510
510
|
}
|
|
@@ -577,7 +577,7 @@ export class BinLogStream {
|
|
|
577
577
|
|
|
578
578
|
let table = this.tableCache.get(tableId);
|
|
579
579
|
if (table == null) {
|
|
580
|
-
// This is an insert for a new table that matches a table in the sync
|
|
580
|
+
// This is an insert for a new table that matches a table in the sync config
|
|
581
581
|
// We need to create the table in the storage and cache it.
|
|
582
582
|
table = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema);
|
|
583
583
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { NormalizedMySQLConnectionConfig } from '../types/types.js';
|
|
2
|
-
import mysqlPromise from 'mysql2/promise';
|
|
3
|
-
import mysql, { FieldPacket, RowDataPacket } from 'mysql2';
|
|
4
|
-
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
5
1
|
import { BaseObserver, logger } from '@powersync/lib-services-framework';
|
|
6
2
|
import { ZongJi } from '@powersync/mysql-zongji';
|
|
3
|
+
import mysql, { FieldPacket, RowDataPacket } from 'mysql2';
|
|
4
|
+
import mysqlPromise from 'mysql2/promise';
|
|
5
|
+
import { NormalizedMySQLConnectionConfig } from '../types/types.js';
|
|
6
|
+
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
7
7
|
|
|
8
8
|
export interface MySQLConnectionManagerListener {
|
|
9
9
|
onEnded(): void;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import mysql from 'mysql2/promise';
|
|
3
|
-
import { MySQLConnectionManager } from './MySQLConnectionManager.js';
|
|
4
3
|
import { ResolvedConnectionConfig } from '../types/types.js';
|
|
4
|
+
import { MySQLConnectionManager } from './MySQLConnectionManager.js';
|
|
5
5
|
|
|
6
6
|
export class MySQLConnectionManagerFactory {
|
|
7
7
|
private readonly connectionManagers = new Set<MySQLConnectionManager>();
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import * as common from '../../common/common-index.js';
|
|
2
|
-
import async from 'async';
|
|
3
|
-
import { BinLogEvent, BinLogQueryEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji';
|
|
4
|
-
import * as zongji_utils from './zongji-utils.js';
|
|
5
1
|
import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
2
|
+
import { BinLogEvent, BinLogQueryEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji';
|
|
3
|
+
import { TablePattern } from '@powersync/service-sync-rules';
|
|
4
|
+
import async from 'async';
|
|
8
5
|
import pkg, {
|
|
9
6
|
AST,
|
|
10
7
|
BaseFrom,
|
|
@@ -13,6 +10,8 @@ import pkg, {
|
|
|
13
10
|
RenameStatement,
|
|
14
11
|
TruncateStatement
|
|
15
12
|
} from 'node-sql-parser';
|
|
13
|
+
import timers from 'timers/promises';
|
|
14
|
+
import * as common from '../../common/common-index.js';
|
|
16
15
|
import {
|
|
17
16
|
isAlterTable,
|
|
18
17
|
isColumnExpression,
|
|
@@ -25,7 +24,8 @@ import {
|
|
|
25
24
|
isTruncate,
|
|
26
25
|
matchedSchemaChangeQuery
|
|
27
26
|
} from '../../utils/parser-utils.js';
|
|
28
|
-
import {
|
|
27
|
+
import { MySQLConnectionManager } from '../MySQLConnectionManager.js';
|
|
28
|
+
import * as zongji_utils from './zongji-utils.js';
|
|
29
29
|
|
|
30
30
|
const { Parser } = pkg;
|
|
31
31
|
|
|
@@ -445,8 +445,8 @@ export class BinLogListener {
|
|
|
445
445
|
} catch (error) {
|
|
446
446
|
if (matchedSchemaChangeQuery(query, Object.values(this.databaseFilter))) {
|
|
447
447
|
this.logger.warn(
|
|
448
|
-
`Failed to parse query: [${query}].
|
|
449
|
-
|
|
448
|
+
`Failed to parse query: [${query}].
|
|
449
|
+
Please review for the schema changes and manually redeploy the sync config if required.`
|
|
450
450
|
);
|
|
451
451
|
}
|
|
452
452
|
return [];
|
|
@@ -537,7 +537,7 @@ export class BinLogListener {
|
|
|
537
537
|
}
|
|
538
538
|
|
|
539
539
|
private createDatabaseFilter(sourceTables: TablePattern[]): { [schema: string]: (table: string) => boolean } {
|
|
540
|
-
// Group sync
|
|
540
|
+
// Group sync config tables by schema
|
|
541
541
|
const schemaMap = new Map<string, TablePattern[]>();
|
|
542
542
|
for (const table of sourceTables) {
|
|
543
543
|
if (!schemaMap.has(table.schema)) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BinLogEvent,
|
|
3
3
|
BinLogGTIDLogEvent,
|
|
4
|
-
|
|
4
|
+
BinLogHeartbeatEvent,
|
|
5
|
+
BinLogHeartbeatEvent_V2,
|
|
6
|
+
BinLogQueryEvent,
|
|
5
7
|
BinLogRotationEvent,
|
|
6
|
-
|
|
8
|
+
BinLogRowEvent,
|
|
7
9
|
BinLogRowUpdateEvent,
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
BinLogHeartbeatEvent,
|
|
11
|
-
BinLogHeartbeatEvent_V2
|
|
10
|
+
BinLogTableMapEvent,
|
|
11
|
+
BinLogXidEvent
|
|
12
12
|
} from '@powersync/mysql-zongji';
|
|
13
13
|
|
|
14
14
|
export function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent {
|
package/src/utils/mysql-utils.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import { SourceEntityDescriptor } from '@powersync/service-core';
|
|
2
3
|
import mysql from 'mysql2';
|
|
3
4
|
import mysqlPromise from 'mysql2/promise';
|
|
4
|
-
import * as types from '../types/types.js';
|
|
5
5
|
import { coerce, gte, satisfies } from 'semver';
|
|
6
|
-
import
|
|
6
|
+
import * as types from '../types/types.js';
|
|
7
7
|
|
|
8
8
|
export type RetriedQueryOptions = {
|
|
9
9
|
connection: mysqlPromise.Connection;
|
|
@@ -63,7 +63,7 @@ export function createPool(config: types.NormalizedMySQLConnectionConfig, option
|
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Return a random server id for a given
|
|
66
|
+
* Return a random server id for a given replication stream id.
|
|
67
67
|
* Expected format is: <syncRuleId>00<random number>
|
|
68
68
|
* The max value for server id in MySQL is 2^32 - 1.
|
|
69
69
|
* We use the GTID format to keep track of our position in the binlog, no state is kept by the MySQL server, therefore
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Alter, AST, Create, Drop,
|
|
1
|
+
import { Alter, AST, Create, Drop, DropIndexStatement, RenameStatement, TruncateStatement } from 'node-sql-parser';
|
|
2
2
|
|
|
3
3
|
// We ignore create table statements, since even in the worst case we will pick up the changes when row events for that
|
|
4
4
|
// table are received.
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
|
|
2
|
-
import { BinLogListener, SchemaChange, SchemaChangeType } from '@module/replication/zongji/BinLogListener.js';
|
|
3
1
|
import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
|
|
2
|
+
import { BinLogListener, SchemaChange, SchemaChangeType } from '@module/replication/zongji/BinLogListener.js';
|
|
3
|
+
import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js';
|
|
4
|
+
import { TablePattern } from '@powersync/service-sync-rules';
|
|
5
|
+
import crypto from 'crypto';
|
|
6
|
+
import { v4 as uuid } from 'uuid';
|
|
7
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
|
|
4
8
|
import {
|
|
5
9
|
clearTestDb,
|
|
6
10
|
createBinlogListener,
|
|
@@ -8,10 +12,6 @@ import {
|
|
|
8
12
|
TEST_CONNECTION_OPTIONS,
|
|
9
13
|
TestBinLogEventHandler
|
|
10
14
|
} from './util.js';
|
|
11
|
-
import { v4 as uuid } from 'uuid';
|
|
12
|
-
import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js';
|
|
13
|
-
import crypto from 'crypto';
|
|
14
|
-
import { TablePattern } from '@powersync/service-sync-rules';
|
|
15
15
|
|
|
16
16
|
describe('BinlogListener tests', { timeout: 60_000 }, () => {
|
|
17
17
|
const MAX_QUEUE_CAPACITY_MB = 1;
|
|
@@ -463,9 +463,13 @@ describe('BinlogListener tests', { timeout: 60_000 }, () => {
|
|
|
463
463
|
await insertRows(connectionManager, 1);
|
|
464
464
|
// multi_schema database insert into test_DATA_multi
|
|
465
465
|
await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${uuid()}','test')`);
|
|
466
|
-
|
|
466
|
+
// Wait for both row entries to appear on the binlog. This avoids a rare race condition in MySQL 5.7
|
|
467
|
+
// where the drop table can hang if run immediately after the table was locked by an insert
|
|
468
|
+
await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(2), { timeout: 5000 });
|
|
467
469
|
|
|
470
|
+
await connectionManager.query(`DROP TABLE ${testTable}`);
|
|
468
471
|
await waitForSchemaChanges(1);
|
|
472
|
+
|
|
469
473
|
await binLogListener.stop();
|
|
470
474
|
expect(eventHandler.rowsWritten).toBe(2);
|
|
471
475
|
assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { qualifiedMySQLTable } from '@module/utils/mysql-utils.js';
|
|
1
2
|
import { storage } from '@powersync/service-core';
|
|
2
3
|
import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
|
|
3
4
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
@@ -5,7 +6,6 @@ import { v4 as uuid } from 'uuid';
|
|
|
5
6
|
import { describe, expect, test } from 'vitest';
|
|
6
7
|
import { BinlogStreamTestContext } from './BinlogStreamUtils.js';
|
|
7
8
|
import { createTestDb, describeWithStorage } from './util.js';
|
|
8
|
-
import { qualifiedMySQLTable } from '@module/utils/mysql-utils.js';
|
|
9
9
|
|
|
10
10
|
const BASIC_SYNC_RULES = `
|
|
11
11
|
bucket_definitions:
|
|
@@ -17,8 +17,8 @@ import {
|
|
|
17
17
|
} from '@powersync/service-core';
|
|
18
18
|
import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
|
|
19
19
|
import mysqlPromise from 'mysql2/promise';
|
|
20
|
-
import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
21
20
|
import timers from 'timers/promises';
|
|
21
|
+
import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* Tests operating on the binlog stream need to configure the stream and manage asynchronous
|
|
@@ -82,7 +82,7 @@ export class BinlogStreamTestContext {
|
|
|
82
82
|
async loadNextSyncRules() {
|
|
83
83
|
const syncRules = await this.factory.getNextSyncRulesContent();
|
|
84
84
|
if (syncRules == null) {
|
|
85
|
-
throw new Error(`Next
|
|
85
|
+
throw new Error(`Next replication stream not available`);
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
this.syncRulesContent = syncRules;
|
|
@@ -93,7 +93,7 @@ export class BinlogStreamTestContext {
|
|
|
93
93
|
async loadActiveSyncRules() {
|
|
94
94
|
const syncRules = await this.factory.getActiveSyncRulesContent();
|
|
95
95
|
if (syncRules == null) {
|
|
96
|
-
throw new Error(`Active
|
|
96
|
+
throw new Error(`Active replication stream not available`);
|
|
97
97
|
}
|
|
98
98
|
|
|
99
99
|
this.syncRulesContent = syncRules;
|
|
@@ -104,7 +104,7 @@ export class BinlogStreamTestContext {
|
|
|
104
104
|
|
|
105
105
|
private getSyncRulesContent(): storage.PersistedSyncRulesContent {
|
|
106
106
|
if (this.syncRulesContent == null) {
|
|
107
|
-
throw new Error('Sync
|
|
107
|
+
throw new Error('Sync config not configured - call updateSyncRules() first');
|
|
108
108
|
}
|
|
109
109
|
return this.syncRulesContent;
|
|
110
110
|
}
|
|
@@ -206,7 +206,7 @@ export async function getClientCheckpoint(
|
|
|
206
206
|
const storage = await storageFactory.getActiveStorage();
|
|
207
207
|
const cp = await storage?.getCheckpoint();
|
|
208
208
|
if (cp == null) {
|
|
209
|
-
throw new Error('No
|
|
209
|
+
throw new Error('No replication stream available');
|
|
210
210
|
}
|
|
211
211
|
lastCp = cp;
|
|
212
212
|
if (cp.lsn && cp.lsn >= gtid.comparable) {
|
package/test/src/config.test.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, expect, test } from 'vitest';
|
|
2
1
|
import {
|
|
3
2
|
normalizeConnectionConfig,
|
|
4
3
|
parseMySQLConnectionParam,
|
|
5
4
|
parseMySQLConnectionParams
|
|
6
5
|
} from '@module/types/types.js';
|
|
6
|
+
import { describe, expect, test } from 'vitest';
|
|
7
7
|
|
|
8
8
|
describe('config', () => {
|
|
9
9
|
test('Should resolve database', () => {
|
|
@@ -1,17 +1,16 @@
|
|
|
1
|
+
import * as common from '@module/common/common-index.js';
|
|
2
|
+
import { toColumnDescriptors } from '@module/common/common-index.js';
|
|
3
|
+
import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
|
|
4
|
+
import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js';
|
|
5
|
+
import { BinLogEvent, ZongJi } from '@powersync/mysql-zongji';
|
|
1
6
|
import {
|
|
2
7
|
applyRowContext,
|
|
3
8
|
CompatibilityContext,
|
|
4
9
|
SqliteInputRow,
|
|
5
|
-
SqliteRow,
|
|
6
10
|
TimeValuePrecision
|
|
7
11
|
} from '@powersync/service-sync-rules';
|
|
8
12
|
import { afterAll, describe, expect, test } from 'vitest';
|
|
9
13
|
import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
10
|
-
import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js';
|
|
11
|
-
import * as common from '@module/common/common-index.js';
|
|
12
|
-
import { BinLogEvent, ZongJi } from '@powersync/mysql-zongji';
|
|
13
|
-
import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
|
|
14
|
-
import { toColumnDescriptors } from '@module/common/common-index.js';
|
|
15
14
|
|
|
16
15
|
describe('MySQL Data Types', () => {
|
|
17
16
|
const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { describe, expect, test } from 'vitest';
|
|
2
1
|
import { matchedSchemaChangeQuery } from '@module/utils/parser-utils.js';
|
|
2
|
+
import { describe, expect, test } from 'vitest';
|
|
3
3
|
|
|
4
4
|
describe('MySQL Parser Util Tests', () => {
|
|
5
5
|
test('matchedSchemaChangeQuery function', () => {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { compareIds, putOp, removeOp, test_utils } from '@powersync/service-core-tests';
|
|
2
2
|
import { beforeAll, describe, expect, test } from 'vitest';
|
|
3
3
|
|
|
4
|
-
import { storage } from '@powersync/service-core';
|
|
5
|
-
import { createTestDb, describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
6
|
-
import { BinlogStreamTestContext } from './BinlogStreamUtils.js';
|
|
7
|
-
import timers from 'timers/promises';
|
|
8
4
|
import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
|
|
9
5
|
import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js';
|
|
6
|
+
import { storage } from '@powersync/service-core';
|
|
7
|
+
import timers from 'timers/promises';
|
|
8
|
+
import { BinlogStreamTestContext } from './BinlogStreamUtils.js';
|
|
9
|
+
import { createTestDb, describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
10
10
|
|
|
11
11
|
describe('MySQL Schema Changes', () => {
|
|
12
12
|
describeWithStorage({ timeout: 20_000 }, defineTests);
|