@powersync/service-module-mysql 0.7.4 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dev/docker/mysql/init-scripts/my.cnf +1 -3
  3. package/dist/api/MySQLRouteAPIAdapter.js +11 -3
  4. package/dist/api/MySQLRouteAPIAdapter.js.map +1 -1
  5. package/dist/common/ReplicatedGTID.js +4 -0
  6. package/dist/common/ReplicatedGTID.js.map +1 -1
  7. package/dist/common/common-index.d.ts +1 -2
  8. package/dist/common/common-index.js +1 -2
  9. package/dist/common/common-index.js.map +1 -1
  10. package/dist/common/mysql-to-sqlite.js +4 -0
  11. package/dist/common/mysql-to-sqlite.js.map +1 -1
  12. package/dist/common/schema-utils.d.ts +20 -0
  13. package/dist/common/{get-replication-columns.js → schema-utils.js} +73 -30
  14. package/dist/common/schema-utils.js.map +1 -0
  15. package/dist/replication/BinLogStream.d.ts +9 -6
  16. package/dist/replication/BinLogStream.js +99 -70
  17. package/dist/replication/BinLogStream.js.map +1 -1
  18. package/dist/replication/zongji/BinLogListener.d.ts +52 -5
  19. package/dist/replication/zongji/BinLogListener.js +302 -85
  20. package/dist/replication/zongji/BinLogListener.js.map +1 -1
  21. package/dist/replication/zongji/zongji-utils.d.ts +2 -1
  22. package/dist/replication/zongji/zongji-utils.js +3 -0
  23. package/dist/replication/zongji/zongji-utils.js.map +1 -1
  24. package/dist/types/node-sql-parser-extended-types.d.ts +31 -0
  25. package/dist/types/node-sql-parser-extended-types.js +2 -0
  26. package/dist/types/node-sql-parser-extended-types.js.map +1 -0
  27. package/dist/utils/mysql-utils.d.ts +4 -2
  28. package/dist/utils/mysql-utils.js +15 -3
  29. package/dist/utils/mysql-utils.js.map +1 -1
  30. package/dist/utils/parser-utils.d.ts +16 -0
  31. package/dist/utils/parser-utils.js +58 -0
  32. package/dist/utils/parser-utils.js.map +1 -0
  33. package/package.json +9 -8
  34. package/src/api/MySQLRouteAPIAdapter.ts +11 -3
  35. package/src/common/ReplicatedGTID.ts +6 -1
  36. package/src/common/common-index.ts +1 -2
  37. package/src/common/mysql-to-sqlite.ts +3 -0
  38. package/src/common/{get-replication-columns.ts → schema-utils.ts} +96 -37
  39. package/src/replication/BinLogStream.ts +119 -91
  40. package/src/replication/zongji/BinLogListener.ts +370 -93
  41. package/src/replication/zongji/zongji-utils.ts +6 -1
  42. package/src/types/node-sql-parser-extended-types.ts +25 -0
  43. package/src/utils/mysql-utils.ts +19 -4
  44. package/src/utils/parser-utils.ts +73 -0
  45. package/test/src/BinLogListener.test.ts +415 -32
  46. package/test/src/BinLogStream.test.ts +128 -52
  47. package/test/src/BinlogStreamUtils.ts +12 -2
  48. package/test/src/parser-utils.test.ts +24 -0
  49. package/test/src/schema-changes.test.ts +663 -0
  50. package/test/src/util.ts +6 -0
  51. package/tsconfig.tsbuildinfo +1 -1
  52. package/dist/common/get-replication-columns.d.ts +0 -12
  53. package/dist/common/get-replication-columns.js.map +0 -1
  54. package/dist/common/get-tables-from-pattern.d.ts +0 -7
  55. package/dist/common/get-tables-from-pattern.js +0 -28
  56. package/dist/common/get-tables-from-pattern.js.map +0 -1
  57. package/src/common/get-tables-from-pattern.ts +0 -44
@@ -0,0 +1,73 @@
1
+ import { Alter, AST, Create, Drop, TruncateStatement, RenameStatement, DropIndexStatement } from 'node-sql-parser';
2
+
3
+ // We ignore create table statements, since even in the worst case we will pick up the changes when row events for that
4
+ // table are received.
5
+ const DDL_KEYWORDS = ['alter table', 'drop table', 'truncate table', 'rename table'];
6
+
7
+ /**
8
+ * Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions.
9
+ * @param query
10
+ * @param matchers
11
+ */
12
+ export function matchedSchemaChangeQuery(query: string, matchers: ((table: string) => boolean)[]) {
13
+ // Normalize case and remove backticks for matching
14
+ const normalizedQuery = query.toLowerCase().replace(/`/g, '');
15
+
16
+ const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword));
17
+ if (isDDLQuery) {
18
+ const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/);
19
+ // Check if any matched table names appear in the query
20
+ for (const token of tokens) {
21
+ const matchFound = matchers.some((matcher) => matcher(token));
22
+ if (matchFound) {
23
+ return true;
24
+ }
25
+ }
26
+ }
27
+
28
+ return false;
29
+ }
30
+
31
+ // @ts-ignore
32
+ export function isTruncate(statement: AST): statement is TruncateStatement {
33
+ // @ts-ignore
34
+ return statement.type === 'truncate';
35
+ }
36
+
37
+ // @ts-ignore
38
+ export function isRenameTable(statement: AST): statement is RenameStatement {
39
+ // @ts-ignore
40
+ return statement.type === 'rename';
41
+ }
42
+
43
+ export function isAlterTable(statement: AST): statement is Alter {
44
+ return statement.type === 'alter';
45
+ }
46
+
47
+ export function isRenameExpression(expression: any): boolean {
48
+ return expression.resource === 'table' && expression.action === 'rename';
49
+ }
50
+
51
+ export function isColumnExpression(expression: any): boolean {
52
+ return expression.resource === 'column';
53
+ }
54
+
55
+ export function isConstraintExpression(expression: any): boolean {
56
+ return (
57
+ (expression.resource === 'key' && expression.keyword === 'primary key') ||
58
+ expression.resource === 'constraint' ||
59
+ (expression.resource === 'index' && expression.action === 'drop')
60
+ );
61
+ }
62
+
63
+ export function isDropTable(statement: AST): statement is Drop {
64
+ return statement.type === 'drop' && statement.keyword === 'table';
65
+ }
66
+
67
+ export function isDropIndex(statement: AST): statement is DropIndexStatement {
68
+ return statement.type === 'drop' && statement.keyword === 'index';
69
+ }
70
+
71
+ export function isCreateUniqueIndex(statement: AST): statement is Create {
72
+ return statement.type === 'create' && statement.keyword === 'index' && statement.index_type === 'unique';
73
+ }
@@ -1,12 +1,24 @@
1
- import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest';
2
- import { BinLogEventHandler, BinLogListener, Row } from '@module/replication/zongji/BinLogListener.js';
1
+ import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
2
+ import {
3
+ BinLogEventHandler,
4
+ BinLogListener,
5
+ Row,
6
+ SchemaChange,
7
+ SchemaChangeType
8
+ } from '@module/replication/zongji/BinLogListener.js';
3
9
  import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
4
- import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
10
+ import { clearTestDb, createTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
5
11
  import { v4 as uuid } from 'uuid';
6
12
  import * as common from '@module/common/common-index.js';
7
- import { createRandomServerId } from '@module/utils/mysql-utils.js';
13
+ import {
14
+ createRandomServerId,
15
+ getMySQLVersion,
16
+ qualifiedMySQLTable,
17
+ satisfiesVersion
18
+ } from '@module/utils/mysql-utils.js';
8
19
  import { TableMapEntry } from '@powersync/mysql-zongji';
9
20
  import crypto from 'crypto';
21
+ import { TablePattern } from '@powersync/service-sync-rules';
10
22
 
11
23
  describe('BinlogListener tests', () => {
12
24
  const MAX_QUEUE_CAPACITY_MB = 1;
@@ -18,26 +30,26 @@ describe('BinlogListener tests', () => {
18
30
  let connectionManager: MySQLConnectionManager;
19
31
  let eventHandler: TestBinLogEventHandler;
20
32
  let binLogListener: BinLogListener;
33
+ let isMySQL57: boolean = false;
21
34
 
22
- beforeEach(async () => {
35
+ beforeAll(async () => {
23
36
  connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {});
24
37
  const connection = await connectionManager.getConnection();
25
- await clearTestDb(connection);
26
- await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
38
+ const version = await getMySQLVersion(connection);
39
+ isMySQL57 = satisfiesVersion(version, '5.7.x');
27
40
  connection.release();
28
- const fromGTID = await getFromGTID(connectionManager);
41
+ });
29
42
 
43
+ beforeEach(async () => {
44
+ const connection = await connectionManager.getConnection();
45
+ await clearTestDb(connection);
46
+ await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
47
+ connection.release();
30
48
  eventHandler = new TestBinLogEventHandler();
31
- binLogListener = new BinLogListener({
32
- connectionManager: connectionManager,
33
- eventHandler: eventHandler,
34
- startPosition: fromGTID.position,
35
- includedTables: ['test_DATA'],
36
- serverId: createRandomServerId(1)
37
- });
49
+ binLogListener = await createBinlogListener();
38
50
  });
39
51
 
40
- afterEach(async () => {
52
+ afterAll(async () => {
41
53
  await connectionManager.end();
42
54
  });
43
55
 
@@ -45,17 +57,15 @@ describe('BinlogListener tests', () => {
45
57
  const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
46
58
  const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill');
47
59
 
48
- const startPromise = binLogListener.start();
49
- setTimeout(async () => binLogListener.stop(), 50);
60
+ await binLogListener.start();
61
+ await binLogListener.stop();
50
62
 
51
- await expect(startPromise).resolves.toBeUndefined();
52
63
  expect(stopSpy).toHaveBeenCalled();
53
64
  expect(queueStopSpy).toHaveBeenCalled();
54
65
  });
55
66
 
56
- test('Pause Zongji binlog listener when processing queue reaches maximum memory size', async () => {
57
- const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause');
58
- const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume');
67
+ test('Zongji listener is stopped when processing queue reaches maximum memory size', async () => {
68
+ const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
59
69
 
60
70
  // Pause the event handler to force a backlog on the processing queue
61
71
  eventHandler.pause();
@@ -63,24 +73,24 @@ describe('BinlogListener tests', () => {
63
73
  const ROW_COUNT = 10;
64
74
  await insertRows(connectionManager, ROW_COUNT);
65
75
 
66
- const startPromise = binLogListener.start();
76
+ await binLogListener.start();
67
77
 
68
- // Wait for listener to pause due to queue reaching capacity
69
- await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 });
78
+ // Wait for listener to stop due to queue reaching capacity
79
+ await vi.waitFor(() => expect(stopSpy).toHaveBeenCalled(), { timeout: 5000 });
70
80
 
71
81
  expect(binLogListener.isQueueOverCapacity()).toBeTruthy();
72
82
  // Resume event processing
73
83
  eventHandler.unpause!();
84
+ const restartSpy = vi.spyOn(binLogListener, 'start');
74
85
 
75
86
  await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 });
76
- binLogListener.stop();
77
- await expect(startPromise).resolves.toBeUndefined();
87
+ await binLogListener.stop();
78
88
  // Confirm resume was called after unpausing
79
- expect(resumeSpy).toHaveBeenCalled();
89
+ expect(restartSpy).toHaveBeenCalled();
80
90
  });
81
91
 
82
- test('Binlog events are correctly forwarded to provided binlog events handler', async () => {
83
- const startPromise = binLogListener.start();
92
+ test('Row events: Write, update, delete', async () => {
93
+ await binLogListener.start();
84
94
 
85
95
  const ROW_COUNT = 10;
86
96
  await insertRows(connectionManager, ROW_COUNT);
@@ -93,9 +103,378 @@ describe('BinlogListener tests', () => {
93
103
  await deleteRows(connectionManager);
94
104
  await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 });
95
105
 
96
- binLogListener.stop();
97
- await expect(startPromise).resolves.toBeUndefined();
106
+ await binLogListener.stop();
107
+ });
108
+
109
+ test('Schema change event: Rename table', async () => {
110
+ await binLogListener.start();
111
+ await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`);
112
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
113
+ await binLogListener.stop();
114
+ assertSchemaChange(
115
+ eventHandler.schemaChanges[0],
116
+ SchemaChangeType.RENAME_TABLE,
117
+ connectionManager.databaseName,
118
+ 'test_DATA',
119
+ 'test_DATA_new'
120
+ );
121
+ });
122
+
123
+ test('Schema change event: Rename multiple tables', async () => {
124
+ // RENAME TABLE supports renaming multiple tables in a single statement
125
+ // We generate a schema change event for each table renamed
126
+ await binLogListener.start();
127
+ await connectionManager.query(`RENAME TABLE
128
+ test_DATA TO test_DATA_new,
129
+ test_DATA_new TO test_DATA
130
+ `);
131
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
132
+ await binLogListener.stop();
133
+ assertSchemaChange(
134
+ eventHandler.schemaChanges[0],
135
+ SchemaChangeType.RENAME_TABLE,
136
+ connectionManager.databaseName,
137
+ 'test_DATA'
138
+ );
139
+ // New table name is undefined since the renamed table is not included by the database filter
140
+ expect(eventHandler.schemaChanges[0].newTable).toBeUndefined();
141
+
142
+ assertSchemaChange(
143
+ eventHandler.schemaChanges[1],
144
+ SchemaChangeType.RENAME_TABLE,
145
+ connectionManager.databaseName,
146
+ 'test_DATA_new',
147
+ 'test_DATA'
148
+ );
149
+ });
150
+
151
+ test('Schema change event: Truncate table', async () => {
152
+ await binLogListener.start();
153
+ await connectionManager.query(`TRUNCATE TABLE test_DATA`);
154
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
155
+ await binLogListener.stop();
156
+ assertSchemaChange(
157
+ eventHandler.schemaChanges[0],
158
+ SchemaChangeType.TRUNCATE_TABLE,
159
+ connectionManager.databaseName,
160
+ 'test_DATA'
161
+ );
162
+ });
163
+
164
+ test('Schema change event: Drop table', async () => {
165
+ await binLogListener.start();
166
+ await connectionManager.query(`DROP TABLE test_DATA`);
167
+ await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
168
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
169
+ await binLogListener.stop();
170
+ assertSchemaChange(
171
+ eventHandler.schemaChanges[0],
172
+ SchemaChangeType.DROP_TABLE,
173
+ connectionManager.databaseName,
174
+ 'test_DATA'
175
+ );
176
+ });
177
+
178
+ test('Schema change event: Drop column', async () => {
179
+ await binLogListener.start();
180
+ await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`);
181
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
182
+ await binLogListener.stop();
183
+ assertSchemaChange(
184
+ eventHandler.schemaChanges[0],
185
+ SchemaChangeType.ALTER_TABLE_COLUMN,
186
+ connectionManager.databaseName,
187
+ 'test_DATA'
188
+ );
189
+ });
190
+
191
+ test('Schema change event: Add column', async () => {
192
+ await binLogListener.start();
193
+ await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`);
194
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
195
+ await binLogListener.stop();
196
+ assertSchemaChange(
197
+ eventHandler.schemaChanges[0],
198
+ SchemaChangeType.ALTER_TABLE_COLUMN,
199
+ connectionManager.databaseName,
200
+ 'test_DATA'
201
+ );
202
+ });
203
+
204
+ test('Schema change event: Modify column', async () => {
205
+ await binLogListener.start();
206
+ await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`);
207
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
208
+ await binLogListener.stop();
209
+ assertSchemaChange(
210
+ eventHandler.schemaChanges[0],
211
+ SchemaChangeType.ALTER_TABLE_COLUMN,
212
+ connectionManager.databaseName,
213
+ 'test_DATA'
214
+ );
215
+ });
216
+
217
+ test('Schema change event: Rename column via change statement', async () => {
218
+ await binLogListener.start();
219
+ await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`);
220
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
221
+ await binLogListener.stop();
222
+ assertSchemaChange(
223
+ eventHandler.schemaChanges[0],
224
+ SchemaChangeType.ALTER_TABLE_COLUMN,
225
+ connectionManager.databaseName,
226
+ 'test_DATA'
227
+ );
228
+ });
229
+
230
+ test('Schema change event: Rename column via rename statement', async () => {
231
+ // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0
232
+ if (!isMySQL57) {
233
+ await binLogListener.start();
234
+ await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`);
235
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
236
+ await binLogListener.stop();
237
+ assertSchemaChange(
238
+ eventHandler.schemaChanges[0],
239
+ SchemaChangeType.ALTER_TABLE_COLUMN,
240
+ connectionManager.databaseName,
241
+ 'test_DATA'
242
+ );
243
+ }
244
+ });
245
+
246
+ test('Schema change event: Multiple column changes', async () => {
247
+ // ALTER TABLE can have multiple column changes in a single statement
248
+ await binLogListener.start();
249
+ await connectionManager.query(
250
+ `ALTER TABLE test_DATA DROP COLUMN description, ADD COLUMN new_description TEXT, MODIFY COLUMN id VARCHAR(50)`
251
+ );
252
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(3), { timeout: 5000 });
253
+ await binLogListener.stop();
254
+ assertSchemaChange(
255
+ eventHandler.schemaChanges[0],
256
+ SchemaChangeType.ALTER_TABLE_COLUMN,
257
+ connectionManager.databaseName,
258
+ 'test_DATA'
259
+ );
260
+
261
+ assertSchemaChange(
262
+ eventHandler.schemaChanges[1],
263
+ SchemaChangeType.ALTER_TABLE_COLUMN,
264
+ connectionManager.databaseName,
265
+ 'test_DATA'
266
+ );
267
+
268
+ assertSchemaChange(
269
+ eventHandler.schemaChanges[2],
270
+ SchemaChangeType.ALTER_TABLE_COLUMN,
271
+ connectionManager.databaseName,
272
+ 'test_DATA'
273
+ );
274
+ });
275
+
276
+ test('Schema change event: Drop and Add primary key', async () => {
277
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
278
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
279
+ binLogListener = await createBinlogListener(sourceTables);
280
+ await binLogListener.start();
281
+ await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`);
282
+ await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`);
283
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
284
+ await binLogListener.stop();
285
+ // Event for the add
286
+ assertSchemaChange(
287
+ eventHandler.schemaChanges[0],
288
+ SchemaChangeType.REPLICATION_IDENTITY,
289
+ connectionManager.databaseName,
290
+ 'test_constraints'
291
+ );
292
+ // Event for the drop
293
+ assertSchemaChange(
294
+ eventHandler.schemaChanges[1],
295
+ SchemaChangeType.REPLICATION_IDENTITY,
296
+ connectionManager.databaseName,
297
+ 'test_constraints'
298
+ );
299
+ });
300
+
301
+ test('Schema change event: Add and drop unique constraint', async () => {
302
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
303
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
304
+ binLogListener = await createBinlogListener(sourceTables);
305
+ await binLogListener.start();
306
+ await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`);
307
+ await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`);
308
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
309
+ await binLogListener.stop();
310
+ // Event for the creation
311
+ assertSchemaChange(
312
+ eventHandler.schemaChanges[0],
313
+ SchemaChangeType.REPLICATION_IDENTITY,
314
+ connectionManager.databaseName,
315
+ 'test_constraints'
316
+ );
317
+ // Event for the drop
318
+ assertSchemaChange(
319
+ eventHandler.schemaChanges[1],
320
+ SchemaChangeType.REPLICATION_IDENTITY,
321
+ connectionManager.databaseName,
322
+ 'test_constraints'
323
+ );
324
+ });
325
+
326
+ test('Schema change event: Add and drop a unique index', async () => {
327
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
328
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
329
+ binLogListener = await createBinlogListener(sourceTables);
330
+ await binLogListener.start();
331
+ await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`);
332
+ await connectionManager.query(`DROP INDEX description_idx ON test_constraints`);
333
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
334
+ await binLogListener.stop();
335
+ // Event for the creation
336
+ assertSchemaChange(
337
+ eventHandler.schemaChanges[0],
338
+ SchemaChangeType.REPLICATION_IDENTITY,
339
+ connectionManager.databaseName,
340
+ 'test_constraints'
341
+ );
342
+ // Event for the drop
343
+ assertSchemaChange(
344
+ eventHandler.schemaChanges[1],
345
+ SchemaChangeType.REPLICATION_IDENTITY,
346
+ connectionManager.databaseName,
347
+ 'test_constraints'
348
+ );
349
+ });
350
+
351
+ test('Schema changes for non-matching tables are ignored', async () => {
352
+ // TableFilter = only match 'test_DATA'
353
+ await binLogListener.start();
354
+ await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`);
355
+ await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`);
356
+ await connectionManager.query(`DROP TABLE test_ignored`);
357
+
358
+ // "Anchor" event to latch onto, ensuring that the schema change events have finished
359
+ await insertRows(connectionManager, 1);
360
+ await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 });
361
+ await binLogListener.stop();
362
+
363
+ expect(eventHandler.schemaChanges.length).toBe(0);
98
364
  });
365
+
366
+ test('Sequential schema change handling', async () => {
367
+ // If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once
368
+ // all the schema changes have been processed
369
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_multiple')];
370
+ binLogListener = await createBinlogListener(sourceTables);
371
+
372
+ await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`);
373
+ await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`);
374
+ await connectionManager.query(`ALTER TABLE test_multiple ADD PRIMARY KEY (id)`);
375
+ await connectionManager.query(`ALTER TABLE test_multiple MODIFY COLUMN new_column TEXT`);
376
+ await connectionManager.query(`DROP TABLE test_multiple`);
377
+
378
+ await binLogListener.start();
379
+
380
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(4), { timeout: 5000 });
381
+ await binLogListener.stop();
382
+ expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
383
+ expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY);
384
+ expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
385
+ expect(eventHandler.schemaChanges[3].type).toBe(SchemaChangeType.DROP_TABLE);
386
+ });
387
+
388
+ test('Unprocessed binlog event received that does match the current table schema', async () => {
389
+ // If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error
390
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
391
+ binLogListener = await createBinlogListener(sourceTables);
392
+
393
+ await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
394
+ await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
395
+ await connectionManager.query(`ALTER TABLE test_failure DROP COLUMN description`);
396
+
397
+ await binLogListener.start();
398
+
399
+ await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(
400
+ /that does not match its current schema/
401
+ );
402
+ });
403
+
404
+ test('Unprocessed binlog event received for a dropped table', async () => {
405
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
406
+ binLogListener = await createBinlogListener(sourceTables);
407
+
408
+ // If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error
409
+ await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
410
+ await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
411
+ await connectionManager.query(`DROP TABLE test_failure`);
412
+
413
+ await binLogListener.start();
414
+
415
+ await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(/or the table has been dropped/);
416
+ });
417
+
418
+ test('Multi database events', async () => {
419
+ await createTestDb(connectionManager, 'multi_schema');
420
+ const testTable = qualifiedMySQLTable('test_DATA_multi', 'multi_schema');
421
+ await connectionManager.query(`CREATE TABLE ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);`);
422
+
423
+ const sourceTables = [
424
+ new TablePattern(connectionManager.databaseName, 'test_DATA'),
425
+ new TablePattern('multi_schema', 'test_DATA_multi')
426
+ ];
427
+ binLogListener = await createBinlogListener(sourceTables);
428
+ await binLogListener.start();
429
+
430
+ // Default database insert into test_DATA
431
+ await insertRows(connectionManager, 1);
432
+ // multi_schema database insert into test_DATA_multi
433
+ await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${uuid()}','test')`);
434
+ await connectionManager.query(`DROP TABLE ${testTable}`);
435
+
436
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
437
+ await binLogListener.stop();
438
+ expect(eventHandler.rowsWritten).toBe(2);
439
+ assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi');
440
+ });
441
+
442
+ async function createBinlogListener(
443
+ sourceTables?: TablePattern[],
444
+ startPosition?: common.BinLogPosition
445
+ ): Promise<BinLogListener> {
446
+ if (!sourceTables) {
447
+ sourceTables = [new TablePattern(connectionManager.databaseName, 'test_DATA')];
448
+ }
449
+
450
+ if (!startPosition) {
451
+ const fromGTID = await getFromGTID(connectionManager);
452
+ startPosition = fromGTID.position;
453
+ }
454
+
455
+ return new BinLogListener({
456
+ connectionManager: connectionManager,
457
+ eventHandler: eventHandler,
458
+ startPosition: startPosition,
459
+ sourceTables: sourceTables,
460
+ serverId: createRandomServerId(1)
461
+ });
462
+ }
463
+
464
+ function assertSchemaChange(
465
+ change: SchemaChange,
466
+ type: SchemaChangeType,
467
+ schema: string,
468
+ table: string,
469
+ newTable?: string
470
+ ) {
471
+ expect(change.type).toBe(type);
472
+ expect(change.schema).toBe(schema);
473
+ expect(change.table).toEqual(table);
474
+ if (newTable) {
475
+ expect(change.newTable).toEqual(newTable);
476
+ }
477
+ }
99
478
  });
100
479
 
101
480
  async function getFromGTID(connectionManager: MySQLConnectionManager) {
@@ -127,6 +506,7 @@ class TestBinLogEventHandler implements BinLogEventHandler {
127
506
  rowsUpdated = 0;
128
507
  rowsDeleted = 0;
129
508
  commitCount = 0;
509
+ schemaChanges: SchemaChange[] = [];
130
510
 
131
511
  unpause: ((value: void | PromiseLike<void>) => void) | undefined;
132
512
  private pausedPromise: Promise<void> | undefined;
@@ -156,6 +536,9 @@ class TestBinLogEventHandler implements BinLogEventHandler {
156
536
  this.commitCount++;
157
537
  }
158
538
 
539
+ async onSchemaChange(change: SchemaChange) {
540
+ this.schemaChanges.push(change);
541
+ }
159
542
  async onTransactionStart(options: { timestamp: Date }) {}
160
543
  async onRotate() {}
161
544
  }