@powersync/service-module-mysql 0.7.4 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/LICENSE +3 -3
  3. package/dev/docker/mysql/init-scripts/my.cnf +1 -3
  4. package/dist/api/MySQLRouteAPIAdapter.js +12 -4
  5. package/dist/api/MySQLRouteAPIAdapter.js.map +1 -1
  6. package/dist/common/ReplicatedGTID.js +4 -0
  7. package/dist/common/ReplicatedGTID.js.map +1 -1
  8. package/dist/common/common-index.d.ts +1 -2
  9. package/dist/common/common-index.js +1 -2
  10. package/dist/common/common-index.js.map +1 -1
  11. package/dist/common/mysql-to-sqlite.d.ts +1 -1
  12. package/dist/common/mysql-to-sqlite.js +4 -0
  13. package/dist/common/mysql-to-sqlite.js.map +1 -1
  14. package/dist/common/schema-utils.d.ts +20 -0
  15. package/dist/common/{get-replication-columns.js → schema-utils.js} +73 -30
  16. package/dist/common/schema-utils.js.map +1 -0
  17. package/dist/replication/BinLogReplicationJob.js +4 -1
  18. package/dist/replication/BinLogReplicationJob.js.map +1 -1
  19. package/dist/replication/BinLogStream.d.ts +9 -6
  20. package/dist/replication/BinLogStream.js +117 -73
  21. package/dist/replication/BinLogStream.js.map +1 -1
  22. package/dist/replication/zongji/BinLogListener.d.ts +60 -6
  23. package/dist/replication/zongji/BinLogListener.js +347 -89
  24. package/dist/replication/zongji/BinLogListener.js.map +1 -1
  25. package/dist/replication/zongji/zongji-utils.d.ts +4 -1
  26. package/dist/replication/zongji/zongji-utils.js +9 -0
  27. package/dist/replication/zongji/zongji-utils.js.map +1 -1
  28. package/dist/types/node-sql-parser-extended-types.d.ts +31 -0
  29. package/dist/types/node-sql-parser-extended-types.js +2 -0
  30. package/dist/types/node-sql-parser-extended-types.js.map +1 -0
  31. package/dist/utils/mysql-utils.d.ts +4 -2
  32. package/dist/utils/mysql-utils.js +15 -3
  33. package/dist/utils/mysql-utils.js.map +1 -1
  34. package/dist/utils/parser-utils.d.ts +16 -0
  35. package/dist/utils/parser-utils.js +58 -0
  36. package/dist/utils/parser-utils.js.map +1 -0
  37. package/package.json +12 -11
  38. package/src/api/MySQLRouteAPIAdapter.ts +15 -4
  39. package/src/common/ReplicatedGTID.ts +6 -1
  40. package/src/common/common-index.ts +1 -2
  41. package/src/common/mysql-to-sqlite.ts +7 -1
  42. package/src/common/{get-replication-columns.ts → schema-utils.ts} +96 -37
  43. package/src/replication/BinLogReplicationJob.ts +4 -1
  44. package/src/replication/BinLogStream.ts +139 -94
  45. package/src/replication/zongji/BinLogListener.ts +421 -100
  46. package/src/replication/zongji/zongji-utils.ts +16 -1
  47. package/src/types/node-sql-parser-extended-types.ts +25 -0
  48. package/src/utils/mysql-utils.ts +19 -4
  49. package/src/utils/parser-utils.ts +73 -0
  50. package/test/src/BinLogListener.test.ts +421 -77
  51. package/test/src/BinLogStream.test.ts +128 -52
  52. package/test/src/BinlogStreamUtils.ts +12 -2
  53. package/test/src/mysql-to-sqlite.test.ts +5 -5
  54. package/test/src/parser-utils.test.ts +24 -0
  55. package/test/src/schema-changes.test.ts +659 -0
  56. package/test/src/util.ts +87 -1
  57. package/tsconfig.tsbuildinfo +1 -1
  58. package/dist/common/get-replication-columns.d.ts +0 -12
  59. package/dist/common/get-replication-columns.js.map +0 -1
  60. package/dist/common/get-tables-from-pattern.d.ts +0 -7
  61. package/dist/common/get-tables-from-pattern.js +0 -28
  62. package/dist/common/get-tables-from-pattern.js.map +0 -1
  63. package/src/common/get-tables-from-pattern.ts +0 -44
@@ -1,12 +1,17 @@
1
- import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest';
2
- import { BinLogEventHandler, BinLogListener, Row } from '@module/replication/zongji/BinLogListener.js';
1
+ import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
2
+ import { BinLogListener, SchemaChange, SchemaChangeType } from '@module/replication/zongji/BinLogListener.js';
3
3
  import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js';
4
- import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
4
+ import {
5
+ clearTestDb,
6
+ createBinlogListener,
7
+ createTestDb,
8
+ TEST_CONNECTION_OPTIONS,
9
+ TestBinLogEventHandler
10
+ } from './util.js';
5
11
  import { v4 as uuid } from 'uuid';
6
- import * as common from '@module/common/common-index.js';
7
- import { createRandomServerId } from '@module/utils/mysql-utils.js';
8
- import { TableMapEntry } from '@powersync/mysql-zongji';
12
+ import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js';
9
13
  import crypto from 'crypto';
14
+ import { TablePattern } from '@powersync/service-sync-rules';
10
15
 
11
16
  describe('BinlogListener tests', () => {
12
17
  const MAX_QUEUE_CAPACITY_MB = 1;
@@ -18,26 +23,30 @@ describe('BinlogListener tests', () => {
18
23
  let connectionManager: MySQLConnectionManager;
19
24
  let eventHandler: TestBinLogEventHandler;
20
25
  let binLogListener: BinLogListener;
26
+ let isMySQL57: boolean = false;
21
27
 
22
- beforeEach(async () => {
28
+ beforeAll(async () => {
23
29
  connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {});
24
30
  const connection = await connectionManager.getConnection();
25
- await clearTestDb(connection);
26
- await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
31
+ const version = await getMySQLVersion(connection);
32
+ isMySQL57 = satisfiesVersion(version, '5.7.x');
27
33
  connection.release();
28
- const fromGTID = await getFromGTID(connectionManager);
34
+ });
29
35
 
36
+ beforeEach(async () => {
37
+ const connection = await connectionManager.getConnection();
38
+ await clearTestDb(connection);
39
+ await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
40
+ connection.release();
30
41
  eventHandler = new TestBinLogEventHandler();
31
- binLogListener = new BinLogListener({
32
- connectionManager: connectionManager,
33
- eventHandler: eventHandler,
34
- startPosition: fromGTID.position,
35
- includedTables: ['test_DATA'],
36
- serverId: createRandomServerId(1)
42
+ binLogListener = await createBinlogListener({
43
+ connectionManager,
44
+ sourceTables: [new TablePattern(connectionManager.databaseName, 'test_DATA')],
45
+ eventHandler
37
46
  });
38
47
  });
39
48
 
40
- afterEach(async () => {
49
+ afterAll(async () => {
41
50
  await connectionManager.end();
42
51
  });
43
52
 
@@ -45,17 +54,15 @@ describe('BinlogListener tests', () => {
45
54
  const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
46
55
  const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill');
47
56
 
48
- const startPromise = binLogListener.start();
49
- setTimeout(async () => binLogListener.stop(), 50);
57
+ await binLogListener.start();
58
+ await binLogListener.stop();
50
59
 
51
- await expect(startPromise).resolves.toBeUndefined();
52
60
  expect(stopSpy).toHaveBeenCalled();
53
61
  expect(queueStopSpy).toHaveBeenCalled();
54
62
  });
55
63
 
56
- test('Pause Zongji binlog listener when processing queue reaches maximum memory size', async () => {
57
- const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause');
58
- const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume');
64
+ test('Zongji listener is stopped when processing queue reaches maximum memory size', async () => {
65
+ const stopSpy = vi.spyOn(binLogListener.zongji, 'stop');
59
66
 
60
67
  // Pause the event handler to force a backlog on the processing queue
61
68
  eventHandler.pause();
@@ -63,24 +70,24 @@ describe('BinlogListener tests', () => {
63
70
  const ROW_COUNT = 10;
64
71
  await insertRows(connectionManager, ROW_COUNT);
65
72
 
66
- const startPromise = binLogListener.start();
73
+ await binLogListener.start();
67
74
 
68
- // Wait for listener to pause due to queue reaching capacity
69
- await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 });
75
+ // Wait for listener to stop due to queue reaching capacity
76
+ await vi.waitFor(() => expect(stopSpy).toHaveBeenCalled(), { timeout: 5000 });
70
77
 
71
78
  expect(binLogListener.isQueueOverCapacity()).toBeTruthy();
72
79
  // Resume event processing
73
80
  eventHandler.unpause!();
81
+ const restartSpy = vi.spyOn(binLogListener, 'start');
74
82
 
75
83
  await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 });
76
- binLogListener.stop();
77
- await expect(startPromise).resolves.toBeUndefined();
84
+ await binLogListener.stop();
78
85
  // Confirm resume was called after unpausing
79
- expect(resumeSpy).toHaveBeenCalled();
86
+ expect(restartSpy).toHaveBeenCalled();
80
87
  });
81
88
 
82
- test('Binlog events are correctly forwarded to provided binlog events handler', async () => {
83
- const startPromise = binLogListener.start();
89
+ test('Row events: Write, update, delete', async () => {
90
+ await binLogListener.start();
84
91
 
85
92
  const ROW_COUNT = 10;
86
93
  await insertRows(connectionManager, ROW_COUNT);
@@ -93,69 +100,406 @@ describe('BinlogListener tests', () => {
93
100
  await deleteRows(connectionManager);
94
101
  await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 });
95
102
 
96
- binLogListener.stop();
97
- await expect(startPromise).resolves.toBeUndefined();
103
+ await binLogListener.stop();
98
104
  });
99
- });
100
105
 
101
- async function getFromGTID(connectionManager: MySQLConnectionManager) {
102
- const connection = await connectionManager.getConnection();
103
- const fromGTID = await common.readExecutedGtid(connection);
104
- connection.release();
106
+ test('Keepalive event', async () => {
107
+ binLogListener.options.keepAliveInactivitySeconds = 1;
108
+ await binLogListener.start();
109
+ await vi.waitFor(() => expect(eventHandler.lastKeepAlive).toBeDefined(), { timeout: 10000 });
110
+ await binLogListener.stop();
111
+ expect(eventHandler.lastKeepAlive).toEqual(binLogListener.options.startGTID.comparable);
112
+ });
105
113
 
106
- return fromGTID;
107
- }
114
+ test('Schema change event: Rename table', async () => {
115
+ await binLogListener.start();
116
+ await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`);
117
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
118
+ await binLogListener.stop();
119
+ assertSchemaChange(
120
+ eventHandler.schemaChanges[0],
121
+ SchemaChangeType.RENAME_TABLE,
122
+ connectionManager.databaseName,
123
+ 'test_DATA',
124
+ 'test_DATA_new'
125
+ );
126
+ });
108
127
 
109
- async function insertRows(connectionManager: MySQLConnectionManager, count: number) {
110
- for (let i = 0; i < count; i++) {
128
+ test('Schema change event: Rename multiple tables', async () => {
129
+ // RENAME TABLE supports renaming multiple tables in a single statement
130
+ // We generate a schema change event for each table renamed
131
+ await binLogListener.start();
132
+ await connectionManager.query(`RENAME TABLE
133
+ test_DATA TO test_DATA_new,
134
+ test_DATA_new TO test_DATA
135
+ `);
136
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
137
+ await binLogListener.stop();
138
+ assertSchemaChange(
139
+ eventHandler.schemaChanges[0],
140
+ SchemaChangeType.RENAME_TABLE,
141
+ connectionManager.databaseName,
142
+ 'test_DATA'
143
+ );
144
+ // New table name is undefined since the renamed table is not included by the database filter
145
+ expect(eventHandler.schemaChanges[0].newTable).toBeUndefined();
146
+
147
+ assertSchemaChange(
148
+ eventHandler.schemaChanges[1],
149
+ SchemaChangeType.RENAME_TABLE,
150
+ connectionManager.databaseName,
151
+ 'test_DATA_new',
152
+ 'test_DATA'
153
+ );
154
+ });
155
+
156
+ test('Schema change event: Truncate table', async () => {
157
+ await binLogListener.start();
158
+ await connectionManager.query(`TRUNCATE TABLE test_DATA`);
159
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
160
+ await binLogListener.stop();
161
+ assertSchemaChange(
162
+ eventHandler.schemaChanges[0],
163
+ SchemaChangeType.TRUNCATE_TABLE,
164
+ connectionManager.databaseName,
165
+ 'test_DATA'
166
+ );
167
+ });
168
+
169
+ test('Schema change event: Drop table', async () => {
170
+ await binLogListener.start();
171
+ await connectionManager.query(`DROP TABLE test_DATA`);
172
+ await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`);
173
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
174
+ await binLogListener.stop();
175
+ assertSchemaChange(
176
+ eventHandler.schemaChanges[0],
177
+ SchemaChangeType.DROP_TABLE,
178
+ connectionManager.databaseName,
179
+ 'test_DATA'
180
+ );
181
+ });
182
+
183
+ test('Schema change event: Drop column', async () => {
184
+ await binLogListener.start();
185
+ await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`);
186
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
187
+ await binLogListener.stop();
188
+ assertSchemaChange(
189
+ eventHandler.schemaChanges[0],
190
+ SchemaChangeType.ALTER_TABLE_COLUMN,
191
+ connectionManager.databaseName,
192
+ 'test_DATA'
193
+ );
194
+ });
195
+
196
+ test('Schema change event: Add column', async () => {
197
+ await binLogListener.start();
198
+ await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`);
199
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
200
+ await binLogListener.stop();
201
+ assertSchemaChange(
202
+ eventHandler.schemaChanges[0],
203
+ SchemaChangeType.ALTER_TABLE_COLUMN,
204
+ connectionManager.databaseName,
205
+ 'test_DATA'
206
+ );
207
+ });
208
+
209
+ test('Schema change event: Modify column', async () => {
210
+ await binLogListener.start();
211
+ await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`);
212
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
213
+ await binLogListener.stop();
214
+ assertSchemaChange(
215
+ eventHandler.schemaChanges[0],
216
+ SchemaChangeType.ALTER_TABLE_COLUMN,
217
+ connectionManager.databaseName,
218
+ 'test_DATA'
219
+ );
220
+ });
221
+
222
+ test('Schema change event: Rename column via change statement', async () => {
223
+ await binLogListener.start();
224
+ await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`);
225
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
226
+ await binLogListener.stop();
227
+ assertSchemaChange(
228
+ eventHandler.schemaChanges[0],
229
+ SchemaChangeType.ALTER_TABLE_COLUMN,
230
+ connectionManager.databaseName,
231
+ 'test_DATA'
232
+ );
233
+ });
234
+
235
+ test('Schema change event: Rename column via rename statement', async () => {
236
+ // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0
237
+ if (!isMySQL57) {
238
+ await binLogListener.start();
239
+ await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`);
240
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
241
+ await binLogListener.stop();
242
+ assertSchemaChange(
243
+ eventHandler.schemaChanges[0],
244
+ SchemaChangeType.ALTER_TABLE_COLUMN,
245
+ connectionManager.databaseName,
246
+ 'test_DATA'
247
+ );
248
+ }
249
+ });
250
+
251
+ test('Schema change event: Multiple column changes', async () => {
252
+ // ALTER TABLE can have multiple column changes in a single statement
253
+ await binLogListener.start();
111
254
  await connectionManager.query(
112
- `INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i} ${crypto.randomBytes(100_000).toString('hex')}')`
255
+ `ALTER TABLE test_DATA DROP COLUMN description, ADD COLUMN new_description TEXT, MODIFY COLUMN id VARCHAR(50)`
256
+ );
257
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(3), { timeout: 5000 });
258
+ await binLogListener.stop();
259
+ assertSchemaChange(
260
+ eventHandler.schemaChanges[0],
261
+ SchemaChangeType.ALTER_TABLE_COLUMN,
262
+ connectionManager.databaseName,
263
+ 'test_DATA'
113
264
  );
114
- }
115
- }
116
265
 
117
- async function updateRows(connectionManager: MySQLConnectionManager) {
118
- await connectionManager.query(`UPDATE test_DATA SET description='updated'`);
119
- }
266
+ assertSchemaChange(
267
+ eventHandler.schemaChanges[1],
268
+ SchemaChangeType.ALTER_TABLE_COLUMN,
269
+ connectionManager.databaseName,
270
+ 'test_DATA'
271
+ );
120
272
 
121
- async function deleteRows(connectionManager: MySQLConnectionManager) {
122
- await connectionManager.query(`DELETE FROM test_DATA`);
123
- }
273
+ assertSchemaChange(
274
+ eventHandler.schemaChanges[2],
275
+ SchemaChangeType.ALTER_TABLE_COLUMN,
276
+ connectionManager.databaseName,
277
+ 'test_DATA'
278
+ );
279
+ });
124
280
 
125
- class TestBinLogEventHandler implements BinLogEventHandler {
126
- rowsWritten = 0;
127
- rowsUpdated = 0;
128
- rowsDeleted = 0;
129
- commitCount = 0;
281
+ test('Schema change event: Drop and Add primary key', async () => {
282
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
283
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
284
+ binLogListener = await createBinlogListener({
285
+ connectionManager,
286
+ eventHandler,
287
+ sourceTables
288
+ });
289
+ await binLogListener.start();
290
+ await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`);
291
+ await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`);
292
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
293
+ await binLogListener.stop();
294
+ // Event for the add
295
+ assertSchemaChange(
296
+ eventHandler.schemaChanges[0],
297
+ SchemaChangeType.REPLICATION_IDENTITY,
298
+ connectionManager.databaseName,
299
+ 'test_constraints'
300
+ );
301
+ // Event for the drop
302
+ assertSchemaChange(
303
+ eventHandler.schemaChanges[1],
304
+ SchemaChangeType.REPLICATION_IDENTITY,
305
+ connectionManager.databaseName,
306
+ 'test_constraints'
307
+ );
308
+ });
130
309
 
131
- unpause: ((value: void | PromiseLike<void>) => void) | undefined;
132
- private pausedPromise: Promise<void> | undefined;
310
+ test('Schema change event: Add and drop unique constraint', async () => {
311
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
312
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
313
+ binLogListener = await createBinlogListener({
314
+ connectionManager,
315
+ eventHandler,
316
+ sourceTables
317
+ });
318
+ await binLogListener.start();
319
+ await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`);
320
+ await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`);
321
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
322
+ await binLogListener.stop();
323
+ // Event for the creation
324
+ assertSchemaChange(
325
+ eventHandler.schemaChanges[0],
326
+ SchemaChangeType.REPLICATION_IDENTITY,
327
+ connectionManager.databaseName,
328
+ 'test_constraints'
329
+ );
330
+ // Event for the drop
331
+ assertSchemaChange(
332
+ eventHandler.schemaChanges[1],
333
+ SchemaChangeType.REPLICATION_IDENTITY,
334
+ connectionManager.databaseName,
335
+ 'test_constraints'
336
+ );
337
+ });
133
338
 
134
- pause() {
135
- this.pausedPromise = new Promise((resolve) => {
136
- this.unpause = resolve;
339
+ test('Schema change event: Add and drop a unique index', async () => {
340
+ await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`);
341
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')];
342
+ binLogListener = await createBinlogListener({
343
+ connectionManager,
344
+ eventHandler,
345
+ sourceTables
137
346
  });
138
- }
347
+ await binLogListener.start();
348
+ await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`);
349
+ await connectionManager.query(`DROP INDEX description_idx ON test_constraints`);
350
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 });
351
+ await binLogListener.stop();
352
+ // Event for the creation
353
+ assertSchemaChange(
354
+ eventHandler.schemaChanges[0],
355
+ SchemaChangeType.REPLICATION_IDENTITY,
356
+ connectionManager.databaseName,
357
+ 'test_constraints'
358
+ );
359
+ // Event for the drop
360
+ assertSchemaChange(
361
+ eventHandler.schemaChanges[1],
362
+ SchemaChangeType.REPLICATION_IDENTITY,
363
+ connectionManager.databaseName,
364
+ 'test_constraints'
365
+ );
366
+ });
139
367
 
140
- async onWrite(rows: Row[], tableMap: TableMapEntry) {
141
- if (this.pausedPromise) {
142
- await this.pausedPromise;
143
- }
144
- this.rowsWritten = this.rowsWritten + rows.length;
145
- }
368
+ test('Schema changes for non-matching tables are ignored', async () => {
369
+ // TableFilter = only match 'test_DATA'
370
+ await binLogListener.start();
371
+ await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`);
372
+ await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`);
373
+ await connectionManager.query(`DROP TABLE test_ignored`);
146
374
 
147
- async onUpdate(afterRows: Row[], beforeRows: Row[], tableMap: TableMapEntry) {
148
- this.rowsUpdated = this.rowsUpdated + afterRows.length;
149
- }
375
+ // "Anchor" event to latch onto, ensuring that the schema change events have finished
376
+ await insertRows(connectionManager, 1);
377
+ await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 });
378
+ await binLogListener.stop();
379
+
380
+ expect(eventHandler.schemaChanges.length).toBe(0);
381
+ });
382
+
383
+ test('Sequential schema change handling', async () => {
384
+ // If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once
385
+ // all the schema changes have been processed
386
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_multiple')];
387
+ binLogListener = await createBinlogListener({
388
+ connectionManager,
389
+ eventHandler,
390
+ sourceTables
391
+ });
392
+
393
+ await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`);
394
+ await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`);
395
+ await connectionManager.query(`ALTER TABLE test_multiple ADD PRIMARY KEY (id)`);
396
+ await connectionManager.query(`ALTER TABLE test_multiple MODIFY COLUMN new_column TEXT`);
397
+ await connectionManager.query(`DROP TABLE test_multiple`);
398
+
399
+ await binLogListener.start();
400
+
401
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(4), { timeout: 5000 });
402
+ await binLogListener.stop();
403
+ expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
404
+ expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY);
405
+ expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN);
406
+ expect(eventHandler.schemaChanges[3].type).toBe(SchemaChangeType.DROP_TABLE);
407
+ });
408
+
409
+ test('Unprocessed binlog event received that does match the current table schema', async () => {
410
+ // If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error
411
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
412
+ binLogListener = await createBinlogListener({
413
+ connectionManager,
414
+ eventHandler,
415
+ sourceTables
416
+ });
417
+
418
+ await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
419
+ await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
420
+ await connectionManager.query(`ALTER TABLE test_failure DROP COLUMN description`);
421
+
422
+ await binLogListener.start();
150
423
 
151
- async onDelete(rows: Row[], tableMap: TableMapEntry) {
152
- this.rowsDeleted = this.rowsDeleted + rows.length;
424
+ await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(
425
+ /that does not match its current schema/
426
+ );
427
+ });
428
+
429
+ test('Unprocessed binlog event received for a dropped table', async () => {
430
+ const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')];
431
+ binLogListener = await createBinlogListener({
432
+ connectionManager,
433
+ eventHandler,
434
+ sourceTables
435
+ });
436
+
437
+ // If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error
438
+ await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`);
439
+ await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`);
440
+ await connectionManager.query(`DROP TABLE test_failure`);
441
+
442
+ await binLogListener.start();
443
+
444
+ await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(/or the table has been dropped/);
445
+ });
446
+
447
+ test('Multi database events', async () => {
448
+ await createTestDb(connectionManager, 'multi_schema');
449
+ const testTable = qualifiedMySQLTable('test_DATA_multi', 'multi_schema');
450
+ await connectionManager.query(`CREATE TABLE ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);`);
451
+
452
+ const sourceTables = [
453
+ new TablePattern(connectionManager.databaseName, 'test_DATA'),
454
+ new TablePattern('multi_schema', 'test_DATA_multi')
455
+ ];
456
+ binLogListener = await createBinlogListener({
457
+ connectionManager,
458
+ eventHandler,
459
+ sourceTables
460
+ });
461
+ await binLogListener.start();
462
+
463
+ // Default database insert into test_DATA
464
+ await insertRows(connectionManager, 1);
465
+ // multi_schema database insert into test_DATA_multi
466
+ await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${uuid()}','test')`);
467
+ await connectionManager.query(`DROP TABLE ${testTable}`);
468
+
469
+ await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 });
470
+ await binLogListener.stop();
471
+ expect(eventHandler.rowsWritten).toBe(2);
472
+ assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi');
473
+ });
474
+
475
+ function assertSchemaChange(
476
+ change: SchemaChange,
477
+ type: SchemaChangeType,
478
+ schema: string,
479
+ table: string,
480
+ newTable?: string
481
+ ) {
482
+ expect(change.type).toBe(type);
483
+ expect(change.schema).toBe(schema);
484
+ expect(change.table).toEqual(table);
485
+ if (newTable) {
486
+ expect(change.newTable).toEqual(newTable);
487
+ }
153
488
  }
489
+ });
154
490
 
155
- async onCommit(lsn: string) {
156
- this.commitCount++;
491
+ async function insertRows(connectionManager: MySQLConnectionManager, count: number) {
492
+ for (let i = 0; i < count; i++) {
493
+ await connectionManager.query(
494
+ `INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i} ${crypto.randomBytes(100_000).toString('hex')}')`
495
+ );
157
496
  }
497
+ }
158
498
 
159
- async onTransactionStart(options: { timestamp: Date }) {}
160
- async onRotate() {}
499
+ async function updateRows(connectionManager: MySQLConnectionManager) {
500
+ await connectionManager.query(`UPDATE test_DATA SET description='updated'`);
501
+ }
502
+
503
+ async function deleteRows(connectionManager: MySQLConnectionManager) {
504
+ await connectionManager.query(`DELETE FROM test_DATA`);
161
505
  }