@powersync/service-module-mysql 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dev/docker/mysql/init-scripts/my.cnf +1 -3
- package/dist/api/MySQLRouteAPIAdapter.js +11 -3
- package/dist/api/MySQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/ReplicatedGTID.js +4 -0
- package/dist/common/ReplicatedGTID.js.map +1 -1
- package/dist/common/common-index.d.ts +1 -2
- package/dist/common/common-index.js +1 -2
- package/dist/common/common-index.js.map +1 -1
- package/dist/common/mysql-to-sqlite.js +4 -0
- package/dist/common/mysql-to-sqlite.js.map +1 -1
- package/dist/common/schema-utils.d.ts +20 -0
- package/dist/common/{get-replication-columns.js → schema-utils.js} +73 -30
- package/dist/common/schema-utils.js.map +1 -0
- package/dist/replication/BinLogStream.d.ts +9 -6
- package/dist/replication/BinLogStream.js +99 -70
- package/dist/replication/BinLogStream.js.map +1 -1
- package/dist/replication/zongji/BinLogListener.d.ts +52 -5
- package/dist/replication/zongji/BinLogListener.js +302 -85
- package/dist/replication/zongji/BinLogListener.js.map +1 -1
- package/dist/replication/zongji/zongji-utils.d.ts +2 -1
- package/dist/replication/zongji/zongji-utils.js +3 -0
- package/dist/replication/zongji/zongji-utils.js.map +1 -1
- package/dist/types/node-sql-parser-extended-types.d.ts +31 -0
- package/dist/types/node-sql-parser-extended-types.js +2 -0
- package/dist/types/node-sql-parser-extended-types.js.map +1 -0
- package/dist/utils/mysql-utils.d.ts +4 -2
- package/dist/utils/mysql-utils.js +15 -3
- package/dist/utils/mysql-utils.js.map +1 -1
- package/dist/utils/parser-utils.d.ts +16 -0
- package/dist/utils/parser-utils.js +58 -0
- package/dist/utils/parser-utils.js.map +1 -0
- package/package.json +9 -8
- package/src/api/MySQLRouteAPIAdapter.ts +11 -3
- package/src/common/ReplicatedGTID.ts +6 -1
- package/src/common/common-index.ts +1 -2
- package/src/common/mysql-to-sqlite.ts +3 -0
- package/src/common/{get-replication-columns.ts → schema-utils.ts} +96 -37
- package/src/replication/BinLogStream.ts +119 -91
- package/src/replication/zongji/BinLogListener.ts +370 -93
- package/src/replication/zongji/zongji-utils.ts +6 -1
- package/src/types/node-sql-parser-extended-types.ts +25 -0
- package/src/utils/mysql-utils.ts +19 -4
- package/src/utils/parser-utils.ts +73 -0
- package/test/src/BinLogListener.test.ts +415 -32
- package/test/src/BinLogStream.test.ts +128 -52
- package/test/src/BinlogStreamUtils.ts +12 -2
- package/test/src/parser-utils.test.ts +24 -0
- package/test/src/schema-changes.test.ts +663 -0
- package/test/src/util.ts +6 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/common/get-replication-columns.d.ts +0 -12
- package/dist/common/get-replication-columns.js.map +0 -1
- package/dist/common/get-tables-from-pattern.d.ts +0 -7
- package/dist/common/get-tables-from-pattern.js +0 -28
- package/dist/common/get-tables-from-pattern.js.map +0 -1
- package/src/common/get-tables-from-pattern.ts +0 -44
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { Alter, AST, Create, Drop, TruncateStatement, RenameStatement, DropIndexStatement } from 'node-sql-parser';
|
|
2
|
+
|
|
3
|
+
// We ignore create table statements, since even in the worst case we will pick up the changes when row events for that
|
|
4
|
+
// table are received.
|
|
5
|
+
const DDL_KEYWORDS = ['alter table', 'drop table', 'truncate table', 'rename table'];
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions.
|
|
9
|
+
* @param query
|
|
10
|
+
* @param matchers
|
|
11
|
+
*/
|
|
12
|
+
export function matchedSchemaChangeQuery(query: string, matchers: ((table: string) => boolean)[]) {
|
|
13
|
+
// Normalize case and remove backticks for matching
|
|
14
|
+
const normalizedQuery = query.toLowerCase().replace(/`/g, '');
|
|
15
|
+
|
|
16
|
+
const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword));
|
|
17
|
+
if (isDDLQuery) {
|
|
18
|
+
const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/);
|
|
19
|
+
// Check if any matched table names appear in the query
|
|
20
|
+
for (const token of tokens) {
|
|
21
|
+
const matchFound = matchers.some((matcher) => matcher(token));
|
|
22
|
+
if (matchFound) {
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// @ts-ignore
|
|
32
|
+
export function isTruncate(statement: AST): statement is TruncateStatement {
|
|
33
|
+
// @ts-ignore
|
|
34
|
+
return statement.type === 'truncate';
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// @ts-ignore
|
|
38
|
+
export function isRenameTable(statement: AST): statement is RenameStatement {
|
|
39
|
+
// @ts-ignore
|
|
40
|
+
return statement.type === 'rename';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function isAlterTable(statement: AST): statement is Alter {
|
|
44
|
+
return statement.type === 'alter';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function isRenameExpression(expression: any): boolean {
|
|
48
|
+
return expression.resource === 'table' && expression.action === 'rename';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function isColumnExpression(expression: any): boolean {
|
|
52
|
+
return expression.resource === 'column';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function isConstraintExpression(expression: any): boolean {
|
|
56
|
+
return (
|
|
57
|
+
(expression.resource === 'key' && expression.keyword === 'primary key') ||
|
|
58
|
+
expression.resource === 'constraint' ||
|
|
59
|
+
(expression.resource === 'index' && expression.action === 'drop')
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function isDropTable(statement: AST): statement is Drop {
|
|
64
|
+
return statement.type === 'drop' && statement.keyword === 'table';
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function isDropIndex(statement: AST): statement is DropIndexStatement {
|
|
68
|
+
return statement.type === 'drop' && statement.keyword === 'index';
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function isCreateUniqueIndex(statement: AST): statement is Create {
|
|
72
|
+
return statement.type === 'create' && statement.keyword === 'index' && statement.index_type === 'unique';
|
|
73
|
+
}
|
|
@@ -1,12 +1,24 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
|
|
2
|
+
import {
|
|
3
|
+
BinLogEventHandler,
|
|
4
|
+
BinLogListener,
|
|
5
|
+
Row,
|
|
6
|
+
SchemaChange,
|
|
7
|
+
SchemaChangeType
|
|
8
|
+
} from '@module/replication/zongji/BinLogListener.js';
|
|
3
9
|
import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
|
|
4
|
-
import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
10
|
+
import { clearTestDb, createTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
|
|
5
11
|
import { v4 as uuid } from 'uuid';
|
|
6
12
|
import * as common from '@module/common/common-index.js';
|
|
7
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
createRandomServerId,
|
|
15
|
+
getMySQLVersion,
|
|
16
|
+
qualifiedMySQLTable,
|
|
17
|
+
satisfiesVersion
|
|
18
|
+
} from '@module/utils/mysql-utils.js';
|
|
8
19
|
import { TableMapEntry } from '@powersync/mysql-zongji';
|
|
9
20
|
import crypto from 'crypto';
|
|
21
|
+
import { TablePattern } from '@powersync/service-sync-rules';
|
|
10
22
|
|
|
11
23
|
describe('BinlogListener tests', () => {
|
|
12
24
|
const MAX_QUEUE_CAPACITY_MB = 1;
|
|
@@ -18,26 +30,26 @@ describe('BinlogListener tests', () => {
|
|
|
18
30
|
let connectionManager: MySQLConnectionManager;
|
|
19
31
|
let eventHandler: TestBinLogEventHandler;
|
|
20
32
|
let binLogListener: BinLogListener;
|
|
33
|
+
let isMySQL57: boolean = false;
|
|
21
34
|
|
|
22
|
-
|
|
35
|
+
beforeAll(async () => {
|
|
23
36
|
connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {});
|
|
24
37
|
const connection = await connectionManager.getConnection();
|
|
25
|
-
await
|
|
26
|
-
|
|
38
|
+
const version = await getMySQLVersion(connection);
|
|
39
|
+
isMySQL57 = satisfiesVersion(version, '5.7.x');
|
|
27
40
|
connection.release();
|
|
28
|
-
|
|
41
|
+
});
|
|
29
42
|
|
|
43
|
+
beforeEach(async () => {
|
|
44
|
+
const connection = await connectionManager.getConnection();
|
|
45
|
+
await clearTestDb(connection);
|
|
46
|
+
await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
|
|
47
|
+
connection.release();
|
|
30
48
|
eventHandler = new TestBinLogEventHandler();
|
|
31
|
-
binLogListener =
|
|
32
|
-
connectionManager: connectionManager,
|
|
33
|
-
eventHandler: eventHandler,
|
|
34
|
-
startPosition: fromGTID.position,
|
|
35
|
-
includedTables: ['test_DATA'],
|
|
36
|
-
serverId: createRandomServerId(1)
|
|
37
|
-
});
|
|
49
|
+
binLogListener = await createBinlogListener();
|
|
38
50
|
});
|
|
39
51
|
|
|
40
|
-
|
|
52
|
+
afterAll(async () => {
|
|
41
53
|
await connectionManager.end();
|
|
42
54
|
});
|
|
43
55
|
|
|
@@ -45,17 +57,15 @@ describe('BinlogListener tests', () => {
|
|
|
45
57
|
const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
|
|
46
58
|
const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill');
|
|
47
59
|
|
|
48
|
-
|
|
49
|
-
|
|
60
|
+
await binLogListener.start();
|
|
61
|
+
await binLogListener.stop();
|
|
50
62
|
|
|
51
|
-
await expect(startPromise).resolves.toBeUndefined();
|
|
52
63
|
expect(stopSpy).toHaveBeenCalled();
|
|
53
64
|
expect(queueStopSpy).toHaveBeenCalled();
|
|
54
65
|
});
|
|
55
66
|
|
|
56
|
-
test('
|
|
57
|
-
const
|
|
58
|
-
const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume');
|
|
67
|
+
test('Zongji listener is stopped when processing queue reaches maximum memory size', async () => {
|
|
68
|
+
const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
|
|
59
69
|
|
|
60
70
|
// Pause the event handler to force a backlog on the processing queue
|
|
61
71
|
eventHandler.pause();
|
|
@@ -63,24 +73,24 @@ describe('BinlogListener tests', () => {
|
|
|
63
73
|
const ROW_COUNT = 10;
|
|
64
74
|
await insertRows(connectionManager, ROW_COUNT);
|
|
65
75
|
|
|
66
|
-
|
|
76
|
+
await binLogListener.start();
|
|
67
77
|
|
|
68
|
-
// Wait for listener to
|
|
69
|
-
await vi.waitFor(() => expect(
|
|
78
|
+
// Wait for listener to stop due to queue reaching capacity
|
|
79
|
+
await vi.waitFor(() => expect(stopSpy).toHaveBeenCalled(), { timeout: 5000 });
|
|
70
80
|
|
|
71
81
|
expect(binLogListener.isQueueOverCapacity()).toBeTruthy();
|
|
72
82
|
// Resume event processing
|
|
73
83
|
eventHandler.unpause!();
|
|
84
|
+
const restartSpy = vi.spyOn(binLogListener, 'start');
|
|
74
85
|
|
|
75
86
|
await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 });
|
|
76
|
-
binLogListener.stop();
|
|
77
|
-
await expect(startPromise).resolves.toBeUndefined();
|
|
87
|
+
await binLogListener.stop();
|
|
78
88
|
// Confirm resume was called after unpausing
|
|
79
|
-
expect(
|
|
89
|
+
expect(restartSpy).toHaveBeenCalled();
|
|
80
90
|
});
|
|
81
91
|
|
|
82
|
-
test('
|
|
83
|
-
|
|
92
|
+
test('Row events: Write, update, delete', async () => {
|
|
93
|
+
await binLogListener.start();
|
|
84
94
|
|
|
85
95
|
const ROW_COUNT = 10;
|
|
86
96
|
await insertRows(connectionManager, ROW_COUNT);
|
|
@@ -93,9 +103,378 @@ describe('BinlogListener tests', () => {
|
|
|
93
103
|
await deleteRows(connectionManager);
|
|
94
104
|
await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 });
|
|
95
105
|
|
|
96
|
-
binLogListener.stop();
|
|
97
|
-
|
|
106
|
+
await binLogListener.stop();
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test('Schema change event: Rename table', async () => {
|
|
110
|
+
await binLogListener.start();
|
|
111
|
+
await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`);
|
|
112
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
113
|
+
await binLogListener.stop();
|
|
114
|
+
assertSchemaChange(
|
|
115
|
+
eventHandler.schemaChanges[0],
|
|
116
|
+
SchemaChangeType.RENAME_TABLE,
|
|
117
|
+
connectionManager.databaseName,
|
|
118
|
+
'test_DATA',
|
|
119
|
+
'test_DATA_new'
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test('Schema change event: Rename multiple tables', async () => {
|
|
124
|
+
// RENAME TABLE supports renaming multiple tables in a single statement
|
|
125
|
+
// We generate a schema change event for each table renamed
|
|
126
|
+
await binLogListener.start();
|
|
127
|
+
await connectionManager.query(`RENAME TABLE
|
|
128
|
+
test_DATA TO test_DATA_new,
|
|
129
|
+
test_DATA_new TO test_DATA
|
|
130
|
+
`);
|
|
131
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
|
|
132
|
+
await binLogListener.stop();
|
|
133
|
+
assertSchemaChange(
|
|
134
|
+
eventHandler.schemaChanges[0],
|
|
135
|
+
SchemaChangeType.RENAME_TABLE,
|
|
136
|
+
connectionManager.databaseName,
|
|
137
|
+
'test_DATA'
|
|
138
|
+
);
|
|
139
|
+
// New table name is undefined since the renamed table is not included by the database filter
|
|
140
|
+
expect(eventHandler.schemaChanges[0].newTable).toBeUndefined();
|
|
141
|
+
|
|
142
|
+
assertSchemaChange(
|
|
143
|
+
eventHandler.schemaChanges[1],
|
|
144
|
+
SchemaChangeType.RENAME_TABLE,
|
|
145
|
+
connectionManager.databaseName,
|
|
146
|
+
'test_DATA_new',
|
|
147
|
+
'test_DATA'
|
|
148
|
+
);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test('Schema change event: Truncate table', async () => {
|
|
152
|
+
await binLogListener.start();
|
|
153
|
+
await connectionManager.query(`TRUNCATE TABLE test_DATA`);
|
|
154
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
155
|
+
await binLogListener.stop();
|
|
156
|
+
assertSchemaChange(
|
|
157
|
+
eventHandler.schemaChanges[0],
|
|
158
|
+
SchemaChangeType.TRUNCATE_TABLE,
|
|
159
|
+
connectionManager.databaseName,
|
|
160
|
+
'test_DATA'
|
|
161
|
+
);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test('Schema change event: Drop table', async () => {
|
|
165
|
+
await binLogListener.start();
|
|
166
|
+
await connectionManager.query(`DROP TABLE test_DATA`);
|
|
167
|
+
await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
|
|
168
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
169
|
+
await binLogListener.stop();
|
|
170
|
+
assertSchemaChange(
|
|
171
|
+
eventHandler.schemaChanges[0],
|
|
172
|
+
SchemaChangeType.DROP_TABLE,
|
|
173
|
+
connectionManager.databaseName,
|
|
174
|
+
'test_DATA'
|
|
175
|
+
);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test('Schema change event: Drop column', async () => {
|
|
179
|
+
await binLogListener.start();
|
|
180
|
+
await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`);
|
|
181
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
182
|
+
await binLogListener.stop();
|
|
183
|
+
assertSchemaChange(
|
|
184
|
+
eventHandler.schemaChanges[0],
|
|
185
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
186
|
+
connectionManager.databaseName,
|
|
187
|
+
'test_DATA'
|
|
188
|
+
);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
test('Schema change event: Add column', async () => {
|
|
192
|
+
await binLogListener.start();
|
|
193
|
+
await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`);
|
|
194
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
195
|
+
await binLogListener.stop();
|
|
196
|
+
assertSchemaChange(
|
|
197
|
+
eventHandler.schemaChanges[0],
|
|
198
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
199
|
+
connectionManager.databaseName,
|
|
200
|
+
'test_DATA'
|
|
201
|
+
);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test('Schema change event: Modify column', async () => {
|
|
205
|
+
await binLogListener.start();
|
|
206
|
+
await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`);
|
|
207
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
208
|
+
await binLogListener.stop();
|
|
209
|
+
assertSchemaChange(
|
|
210
|
+
eventHandler.schemaChanges[0],
|
|
211
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
212
|
+
connectionManager.databaseName,
|
|
213
|
+
'test_DATA'
|
|
214
|
+
);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
test('Schema change event: Rename column via change statement', async () => {
|
|
218
|
+
await binLogListener.start();
|
|
219
|
+
await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`);
|
|
220
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
221
|
+
await binLogListener.stop();
|
|
222
|
+
assertSchemaChange(
|
|
223
|
+
eventHandler.schemaChanges[0],
|
|
224
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
225
|
+
connectionManager.databaseName,
|
|
226
|
+
'test_DATA'
|
|
227
|
+
);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test('Schema change event: Rename column via rename statement', async () => {
|
|
231
|
+
// Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0
|
|
232
|
+
if (!isMySQL57) {
|
|
233
|
+
await binLogListener.start();
|
|
234
|
+
await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`);
|
|
235
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
236
|
+
await binLogListener.stop();
|
|
237
|
+
assertSchemaChange(
|
|
238
|
+
eventHandler.schemaChanges[0],
|
|
239
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
240
|
+
connectionManager.databaseName,
|
|
241
|
+
'test_DATA'
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test('Schema change event: Multiple column changes', async () => {
|
|
247
|
+
// ALTER TABLE can have multiple column changes in a single statement
|
|
248
|
+
await binLogListener.start();
|
|
249
|
+
await connectionManager.query(
|
|
250
|
+
`ALTER TABLE test_DATA DROP COLUMN description, ADD COLUMN new_description TEXT, MODIFY COLUMN id VARCHAR(50)`
|
|
251
|
+
);
|
|
252
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(3), { timeout: 5000 });
|
|
253
|
+
await binLogListener.stop();
|
|
254
|
+
assertSchemaChange(
|
|
255
|
+
eventHandler.schemaChanges[0],
|
|
256
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
257
|
+
connectionManager.databaseName,
|
|
258
|
+
'test_DATA'
|
|
259
|
+
);
|
|
260
|
+
|
|
261
|
+
assertSchemaChange(
|
|
262
|
+
eventHandler.schemaChanges[1],
|
|
263
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
264
|
+
connectionManager.databaseName,
|
|
265
|
+
'test_DATA'
|
|
266
|
+
);
|
|
267
|
+
|
|
268
|
+
assertSchemaChange(
|
|
269
|
+
eventHandler.schemaChanges[2],
|
|
270
|
+
SchemaChangeType.ALTER_TABLE_COLUMN,
|
|
271
|
+
connectionManager.databaseName,
|
|
272
|
+
'test_DATA'
|
|
273
|
+
);
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
test('Schema change event: Drop and Add primary key', async () => {
|
|
277
|
+
await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
|
|
278
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
|
|
279
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
280
|
+
await binLogListener.start();
|
|
281
|
+
await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`);
|
|
282
|
+
await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`);
|
|
283
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
|
|
284
|
+
await binLogListener.stop();
|
|
285
|
+
// Event for the add
|
|
286
|
+
assertSchemaChange(
|
|
287
|
+
eventHandler.schemaChanges[0],
|
|
288
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
289
|
+
connectionManager.databaseName,
|
|
290
|
+
'test_constraints'
|
|
291
|
+
);
|
|
292
|
+
// Event for the drop
|
|
293
|
+
assertSchemaChange(
|
|
294
|
+
eventHandler.schemaChanges[1],
|
|
295
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
296
|
+
connectionManager.databaseName,
|
|
297
|
+
'test_constraints'
|
|
298
|
+
);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
test('Schema change event: Add and drop unique constraint', async () => {
|
|
302
|
+
await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
|
|
303
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
|
|
304
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
305
|
+
await binLogListener.start();
|
|
306
|
+
await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`);
|
|
307
|
+
await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`);
|
|
308
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
|
|
309
|
+
await binLogListener.stop();
|
|
310
|
+
// Event for the creation
|
|
311
|
+
assertSchemaChange(
|
|
312
|
+
eventHandler.schemaChanges[0],
|
|
313
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
314
|
+
connectionManager.databaseName,
|
|
315
|
+
'test_constraints'
|
|
316
|
+
);
|
|
317
|
+
// Event for the drop
|
|
318
|
+
assertSchemaChange(
|
|
319
|
+
eventHandler.schemaChanges[1],
|
|
320
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
321
|
+
connectionManager.databaseName,
|
|
322
|
+
'test_constraints'
|
|
323
|
+
);
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
test('Schema change event: Add and drop a unique index', async () => {
|
|
327
|
+
await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
|
|
328
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
|
|
329
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
330
|
+
await binLogListener.start();
|
|
331
|
+
await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`);
|
|
332
|
+
await connectionManager.query(`DROP INDEX description_idx ON test_constraints`);
|
|
333
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
|
|
334
|
+
await binLogListener.stop();
|
|
335
|
+
// Event for the creation
|
|
336
|
+
assertSchemaChange(
|
|
337
|
+
eventHandler.schemaChanges[0],
|
|
338
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
339
|
+
connectionManager.databaseName,
|
|
340
|
+
'test_constraints'
|
|
341
|
+
);
|
|
342
|
+
// Event for the drop
|
|
343
|
+
assertSchemaChange(
|
|
344
|
+
eventHandler.schemaChanges[1],
|
|
345
|
+
SchemaChangeType.REPLICATION_IDENTITY,
|
|
346
|
+
connectionManager.databaseName,
|
|
347
|
+
'test_constraints'
|
|
348
|
+
);
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
test('Schema changes for non-matching tables are ignored', async () => {
|
|
352
|
+
// TableFilter = only match 'test_DATA'
|
|
353
|
+
await binLogListener.start();
|
|
354
|
+
await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`);
|
|
355
|
+
await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`);
|
|
356
|
+
await connectionManager.query(`DROP TABLE test_ignored`);
|
|
357
|
+
|
|
358
|
+
// "Anchor" event to latch onto, ensuring that the schema change events have finished
|
|
359
|
+
await insertRows(connectionManager, 1);
|
|
360
|
+
await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 });
|
|
361
|
+
await binLogListener.stop();
|
|
362
|
+
|
|
363
|
+
expect(eventHandler.schemaChanges.length).toBe(0);
|
|
98
364
|
});
|
|
365
|
+
|
|
366
|
+
test('Sequential schema change handling', async () => {
|
|
367
|
+
// If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once
|
|
368
|
+
// all the schema changes have been processed
|
|
369
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_multiple')];
|
|
370
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
371
|
+
|
|
372
|
+
await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`);
|
|
373
|
+
await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`);
|
|
374
|
+
await connectionManager.query(`ALTER TABLE test_multiple ADD PRIMARY KEY (id)`);
|
|
375
|
+
await connectionManager.query(`ALTER TABLE test_multiple MODIFY COLUMN new_column TEXT`);
|
|
376
|
+
await connectionManager.query(`DROP TABLE test_multiple`);
|
|
377
|
+
|
|
378
|
+
await binLogListener.start();
|
|
379
|
+
|
|
380
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(4), { timeout: 5000 });
|
|
381
|
+
await binLogListener.stop();
|
|
382
|
+
expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
|
|
383
|
+
expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY);
|
|
384
|
+
expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
|
|
385
|
+
expect(eventHandler.schemaChanges[3].type).toBe(SchemaChangeType.DROP_TABLE);
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
test('Unprocessed binlog event received that does match the current table schema', async () => {
|
|
389
|
+
// If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error
|
|
390
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
|
|
391
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
392
|
+
|
|
393
|
+
await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
|
|
394
|
+
await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
|
|
395
|
+
await connectionManager.query(`ALTER TABLE test_failure DROP COLUMN description`);
|
|
396
|
+
|
|
397
|
+
await binLogListener.start();
|
|
398
|
+
|
|
399
|
+
await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(
|
|
400
|
+
/that does not match its current schema/
|
|
401
|
+
);
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
test('Unprocessed binlog event received for a dropped table', async () => {
|
|
405
|
+
const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
|
|
406
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
407
|
+
|
|
408
|
+
// If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error
|
|
409
|
+
await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
|
|
410
|
+
await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
|
|
411
|
+
await connectionManager.query(`DROP TABLE test_failure`);
|
|
412
|
+
|
|
413
|
+
await binLogListener.start();
|
|
414
|
+
|
|
415
|
+
await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(/or the table has been dropped/);
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
test('Multi database events', async () => {
|
|
419
|
+
await createTestDb(connectionManager, 'multi_schema');
|
|
420
|
+
const testTable = qualifiedMySQLTable('test_DATA_multi', 'multi_schema');
|
|
421
|
+
await connectionManager.query(`CREATE TABLE ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);`);
|
|
422
|
+
|
|
423
|
+
const sourceTables = [
|
|
424
|
+
new TablePattern(connectionManager.databaseName, 'test_DATA'),
|
|
425
|
+
new TablePattern('multi_schema', 'test_DATA_multi')
|
|
426
|
+
];
|
|
427
|
+
binLogListener = await createBinlogListener(sourceTables);
|
|
428
|
+
await binLogListener.start();
|
|
429
|
+
|
|
430
|
+
// Default database insert into test_DATA
|
|
431
|
+
await insertRows(connectionManager, 1);
|
|
432
|
+
// multi_schema database insert into test_DATA_multi
|
|
433
|
+
await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${uuid()}','test')`);
|
|
434
|
+
await connectionManager.query(`DROP TABLE ${testTable}`);
|
|
435
|
+
|
|
436
|
+
await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
|
|
437
|
+
await binLogListener.stop();
|
|
438
|
+
expect(eventHandler.rowsWritten).toBe(2);
|
|
439
|
+
assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi');
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
async function createBinlogListener(
|
|
443
|
+
sourceTables?: TablePattern[],
|
|
444
|
+
startPosition?: common.BinLogPosition
|
|
445
|
+
): Promise<BinLogListener> {
|
|
446
|
+
if (!sourceTables) {
|
|
447
|
+
sourceTables = [new TablePattern(connectionManager.databaseName, 'test_DATA')];
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
if (!startPosition) {
|
|
451
|
+
const fromGTID = await getFromGTID(connectionManager);
|
|
452
|
+
startPosition = fromGTID.position;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
return new BinLogListener({
|
|
456
|
+
connectionManager: connectionManager,
|
|
457
|
+
eventHandler: eventHandler,
|
|
458
|
+
startPosition: startPosition,
|
|
459
|
+
sourceTables: sourceTables,
|
|
460
|
+
serverId: createRandomServerId(1)
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function assertSchemaChange(
|
|
465
|
+
change: SchemaChange,
|
|
466
|
+
type: SchemaChangeType,
|
|
467
|
+
schema: string,
|
|
468
|
+
table: string,
|
|
469
|
+
newTable?: string
|
|
470
|
+
) {
|
|
471
|
+
expect(change.type).toBe(type);
|
|
472
|
+
expect(change.schema).toBe(schema);
|
|
473
|
+
expect(change.table).toEqual(table);
|
|
474
|
+
if (newTable) {
|
|
475
|
+
expect(change.newTable).toEqual(newTable);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
99
478
|
});
|
|
100
479
|
|
|
101
480
|
async function getFromGTID(connectionManager: MySQLConnectionManager) {
|
|
@@ -127,6 +506,7 @@ class TestBinLogEventHandler implements BinLogEventHandler {
|
|
|
127
506
|
rowsUpdated = 0;
|
|
128
507
|
rowsDeleted = 0;
|
|
129
508
|
commitCount = 0;
|
|
509
|
+
schemaChanges: SchemaChange[] = [];
|
|
130
510
|
|
|
131
511
|
unpause: ((value: void | PromiseLike<void>) => void) | undefined;
|
|
132
512
|
private pausedPromise: Promise<void> | undefined;
|
|
@@ -156,6 +536,9 @@ class TestBinLogEventHandler implements BinLogEventHandler {
|
|
|
156
536
|
this.commitCount++;
|
|
157
537
|
}
|
|
158
538
|
|
|
539
|
+
async onSchemaChange(change: SchemaChange) {
|
|
540
|
+
this.schemaChanges.push(change);
|
|
541
|
+
}
|
|
159
542
|
async onTransactionStart(options: { timestamp: Date }) {}
|
|
160
543
|
async onRotate() {}
|
|
161
544
|
}
|