@powersync/service-module-mysql 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/api/MySQLRouteAPIAdapter.js +3 -3
- package/dist/api/MySQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/ReplicatedGTID.d.ts +16 -9
- package/dist/common/ReplicatedGTID.js +49 -25
- package/dist/common/ReplicatedGTID.js.map +1 -1
- package/dist/common/check-source-configuration.js +11 -0
- package/dist/common/check-source-configuration.js.map +1 -1
- package/dist/common/read-executed-gtid.d.ts +11 -1
- package/dist/common/read-executed-gtid.js +67 -5
- package/dist/common/read-executed-gtid.js.map +1 -1
- package/dist/replication/BinLogStream.d.ts +2 -0
- package/dist/replication/BinLogStream.js +42 -14
- package/dist/replication/BinLogStream.js.map +1 -1
- package/dist/replication/zongji/BinLogListener.d.ts +17 -2
- package/dist/replication/zongji/BinLogListener.js +42 -29
- package/dist/replication/zongji/BinLogListener.js.map +1 -1
- package/package.json +4 -4
- package/src/api/MySQLRouteAPIAdapter.ts +4 -4
- package/src/common/ReplicatedGTID.ts +65 -32
- package/src/common/check-source-configuration.ts +15 -0
- package/src/common/read-executed-gtid.ts +76 -6
- package/src/replication/BinLogStream.ts +45 -14
- package/src/replication/zongji/BinLogListener.ts +61 -29
- package/test/src/BinLogListener.test.ts +18 -0
- package/test/src/ReplicatedGTID.test.ts +138 -0
- package/test/src/check-source-configuration.test.ts +70 -0
- package/test/src/read-executed-gtid.test.ts +188 -0
- package/test/src/util.ts +25 -2
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -2,6 +2,17 @@ import mysqlPromise from 'mysql2/promise';
|
|
|
2
2
|
import * as mysql_utils from '../utils/mysql-utils.js';
|
|
3
3
|
import { ReplicatedGTID } from './ReplicatedGTID.js';
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Gets the `@@server_uuid` of the current connected server
|
|
7
|
+
*/
|
|
8
|
+
export async function readServerUuid(connection: mysqlPromise.Connection): Promise<string> {
|
|
9
|
+
const [[result]] = await mysql_utils.retriedQuery({
|
|
10
|
+
connection,
|
|
11
|
+
query: `SELECT @@server_uuid AS server_uuid`
|
|
12
|
+
});
|
|
13
|
+
return result.server_uuid;
|
|
14
|
+
}
|
|
15
|
+
|
|
5
16
|
/**
|
|
6
17
|
* Gets the current master HEAD GTID
|
|
7
18
|
*/
|
|
@@ -28,21 +39,80 @@ export async function readExecutedGtid(connection: mysqlPromise.Connection): Pro
|
|
|
28
39
|
offset: parseInt(binlogStatus.Position)
|
|
29
40
|
};
|
|
30
41
|
|
|
42
|
+
const activeServerUuid = await readServerUuid(connection);
|
|
43
|
+
const executedGtidSet = binlogStatus.Executed_Gtid_Set.trim();
|
|
44
|
+
|
|
45
|
+
if (executedGtidSet.length === 0) {
|
|
46
|
+
// New server with no transactions executed yet. Keep the current binlog
|
|
47
|
+
// coordinate so this synthetic GTID can still be validated after a restart.
|
|
48
|
+
return new ReplicatedGTID({
|
|
49
|
+
rawGtid: `${activeServerUuid}:0`,
|
|
50
|
+
position
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const gtidSets = executedGtidSet.split(',');
|
|
55
|
+
const latestActiveGtid = await getLatestActiveGtid(gtidSets, activeServerUuid);
|
|
31
56
|
return new ReplicatedGTID({
|
|
32
|
-
|
|
33
|
-
position
|
|
34
|
-
raw_gtid: binlogStatus.Executed_Gtid_Set
|
|
57
|
+
rawGtid: latestActiveGtid,
|
|
58
|
+
position
|
|
35
59
|
});
|
|
36
60
|
}
|
|
37
61
|
|
|
38
|
-
export async function
|
|
62
|
+
export async function getLatestActiveGtid(gtidSets: string[], activeServerUuid: string): Promise<string> {
|
|
63
|
+
for (const gtidSet of gtidSets) {
|
|
64
|
+
const [serverUuid, ...intervals] = gtidSet.trim().split(':');
|
|
65
|
+
if (serverUuid === activeServerUuid) {
|
|
66
|
+
let maxTransactionId: number | null = null;
|
|
67
|
+
for (const interval of intervals) {
|
|
68
|
+
const [start, end] = interval.split('-');
|
|
69
|
+
const startId = parseInt(start, 10);
|
|
70
|
+
const endId = end !== undefined ? parseInt(end, 10) : startId;
|
|
71
|
+
if (!Number.isNaN(startId)) {
|
|
72
|
+
maxTransactionId = Math.max(maxTransactionId ?? 0, startId);
|
|
73
|
+
}
|
|
74
|
+
if (!Number.isNaN(endId)) {
|
|
75
|
+
maxTransactionId = Math.max(maxTransactionId ?? 0, endId);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return activeServerUuid + ':' + maxTransactionId;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return `${activeServerUuid}:0`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Checks that a stored resume GTID is still part of the server's executed history and that its
|
|
87
|
+
* binlog coordinate is still readable. This detects source rewinds where a restored server keeps
|
|
88
|
+
* the same UUID or recreates a binlog with the same filename but a shorter length.
|
|
89
|
+
*/
|
|
90
|
+
export async function isGtidPositionStillAvailable(
|
|
39
91
|
connection: mysqlPromise.Connection,
|
|
40
|
-
|
|
92
|
+
gtid: ReplicatedGTID
|
|
41
93
|
): Promise<boolean> {
|
|
42
94
|
const [logFiles] = await mysql_utils.retriedQuery({
|
|
43
95
|
connection,
|
|
44
96
|
query: `SHOW BINARY LOGS;`
|
|
45
97
|
});
|
|
98
|
+
const logFile = logFiles.find((file) => file['Log_name'] == gtid.position.filename);
|
|
99
|
+
|
|
100
|
+
if (!logFile || Number(logFile['File_size']) < gtid.position.offset) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Transaction zero is PowerSync's synthetic position before the first
|
|
105
|
+
// transaction from this server UUID. It is not valid MySQL GTID_SET syntax,
|
|
106
|
+
// so its availability is determined by the binlog coordinate above.
|
|
107
|
+
if (gtid.raw.split(':')[1] === '0') {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const [[result]] = await mysql_utils.retriedQuery({
|
|
112
|
+
connection,
|
|
113
|
+
query: `SELECT GTID_SUBSET(?, @@GLOBAL.gtid_executed) AS is_executed`,
|
|
114
|
+
params: [gtid.raw]
|
|
115
|
+
});
|
|
46
116
|
|
|
47
|
-
return
|
|
117
|
+
return result.is_executed === 1;
|
|
48
118
|
}
|
|
@@ -73,6 +73,8 @@ export class BinLogStream {
|
|
|
73
73
|
|
|
74
74
|
private readonly logger: Logger;
|
|
75
75
|
|
|
76
|
+
private activeServerUuid: string | null = null;
|
|
77
|
+
|
|
76
78
|
private tableCache = new Map<string | number, storage.SourceTable[]>();
|
|
77
79
|
|
|
78
80
|
private replicationLag = new ReplicationLagTracker();
|
|
@@ -220,16 +222,23 @@ export class BinLogStream {
|
|
|
220
222
|
this.logger.info(`Initial replication already done.`);
|
|
221
223
|
|
|
222
224
|
if (lastKnowGTID) {
|
|
223
|
-
// Check if the specific binlog file is still available. If it isn't, we need to snapshot again.
|
|
224
225
|
const connection = await this.connections.getConnection();
|
|
225
226
|
try {
|
|
226
|
-
|
|
227
|
+
// Check if the active server uuid matches the one in the GTID
|
|
228
|
+
if (this.activeServerUuid !== lastKnowGTID.serverUuid) {
|
|
229
|
+
this.logger.info(
|
|
230
|
+
`The source server uuid has changed. Active server uuid ${this.activeServerUuid} does not match the server uuid from the resume checkpoint: ${lastKnowGTID.serverUuid}, re-snapshotting to ensure consistency.`
|
|
231
|
+
);
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const isAvailable = await common.isGtidPositionStillAvailable(connection, lastKnowGTID);
|
|
227
236
|
if (!isAvailable) {
|
|
228
237
|
this.logger.info(
|
|
229
|
-
`
|
|
238
|
+
`Resume GTID ${lastKnowGTID.raw} at ${lastKnowGTID.position.filename}:${lastKnowGTID.position.offset} is no longer present in the executed GTID history or available BinLogs, re-snapshotting to ensure consistency.`
|
|
230
239
|
);
|
|
240
|
+
return false;
|
|
231
241
|
}
|
|
232
|
-
return isAvailable;
|
|
233
242
|
} finally {
|
|
234
243
|
connection.release();
|
|
235
244
|
}
|
|
@@ -267,7 +276,7 @@ export class BinLogStream {
|
|
|
267
276
|
const flushResults = await this.storage.startBatch(
|
|
268
277
|
{
|
|
269
278
|
logger: this.logger,
|
|
270
|
-
zeroLSN: common.ReplicatedGTID.ZERO.comparable,
|
|
279
|
+
zeroLSN: common.ReplicatedGTID.ZERO(this.activeServerUuid!).comparable,
|
|
271
280
|
defaultSchema: this.defaultSchema,
|
|
272
281
|
storeCurrentData: false
|
|
273
282
|
},
|
|
@@ -363,7 +372,19 @@ export class BinLogStream {
|
|
|
363
372
|
}
|
|
364
373
|
}
|
|
365
374
|
|
|
375
|
+
private async ensureActiveServerUuid() {
|
|
376
|
+
if (this.activeServerUuid == null) {
|
|
377
|
+
const connection = await this.connections.getConnection();
|
|
378
|
+
try {
|
|
379
|
+
this.activeServerUuid = await common.readServerUuid(connection);
|
|
380
|
+
} finally {
|
|
381
|
+
connection.release();
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
366
386
|
async initReplication() {
|
|
387
|
+
await this.ensureActiveServerUuid();
|
|
367
388
|
const connection = await this.connections.getConnection();
|
|
368
389
|
const errors = await common.checkSourceConfiguration(connection);
|
|
369
390
|
connection.release();
|
|
@@ -382,7 +403,7 @@ export class BinLogStream {
|
|
|
382
403
|
await this.storage.startBatch(
|
|
383
404
|
{
|
|
384
405
|
logger: this.logger,
|
|
385
|
-
zeroLSN: common.ReplicatedGTID.ZERO.comparable,
|
|
406
|
+
zeroLSN: common.ReplicatedGTID.ZERO(this.activeServerUuid!).comparable,
|
|
386
407
|
defaultSchema: this.defaultSchema,
|
|
387
408
|
storeCurrentData: false
|
|
388
409
|
},
|
|
@@ -406,21 +427,30 @@ export class BinLogStream {
|
|
|
406
427
|
}
|
|
407
428
|
|
|
408
429
|
async streamChanges() {
|
|
430
|
+
await this.ensureActiveServerUuid();
|
|
409
431
|
const serverId = createRandomServerId(this.storage.replicationStreamId);
|
|
410
432
|
|
|
411
433
|
const connection = await this.connections.getConnection();
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
434
|
+
let fromGTID: common.ReplicatedGTID;
|
|
435
|
+
try {
|
|
436
|
+
const { resumeLsn: resume_lsn } = await this.storage.getStatus();
|
|
437
|
+
if (resume_lsn) {
|
|
438
|
+
this.logger.info(`Existing resume LSN found: ${resume_lsn}`);
|
|
439
|
+
}
|
|
440
|
+
fromGTID = resume_lsn
|
|
441
|
+
? common.ReplicatedGTID.fromSerialized(resume_lsn)
|
|
442
|
+
: await common.readExecutedGtid(connection);
|
|
443
|
+
} finally {
|
|
444
|
+
connection.release();
|
|
415
445
|
}
|
|
416
|
-
const fromGTID = resume_lsn
|
|
417
|
-
? common.ReplicatedGTID.fromSerialized(resume_lsn)
|
|
418
|
-
: await common.readExecutedGtid(connection);
|
|
419
|
-
connection.release();
|
|
420
446
|
|
|
421
447
|
if (!this.stopped) {
|
|
422
448
|
await this.storage.startBatch(
|
|
423
|
-
{
|
|
449
|
+
{
|
|
450
|
+
zeroLSN: common.ReplicatedGTID.ZERO(this.activeServerUuid!).comparable,
|
|
451
|
+
defaultSchema: this.defaultSchema,
|
|
452
|
+
storeCurrentData: false
|
|
453
|
+
},
|
|
424
454
|
async (batch) => {
|
|
425
455
|
const binlogEventHandler = this.createBinlogEventHandler(batch);
|
|
426
456
|
const binlogListener = new BinLogListener({
|
|
@@ -429,6 +459,7 @@ export class BinLogStream {
|
|
|
429
459
|
startGTID: fromGTID,
|
|
430
460
|
connectionManager: this.connections,
|
|
431
461
|
serverId: serverId,
|
|
462
|
+
activeServerUuid: this.activeServerUuid!,
|
|
432
463
|
eventHandler: binlogEventHandler
|
|
433
464
|
});
|
|
434
465
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
1
|
+
import { Logger, ReplicationAssertionError, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { BinLogEvent, BinLogQueryEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji';
|
|
3
3
|
import { TablePattern } from '@powersync/service-sync-rules';
|
|
4
4
|
import async from 'async';
|
|
@@ -76,7 +76,14 @@ export interface BinLogListenerOptions {
|
|
|
76
76
|
connectionManager: MySQLConnectionManager;
|
|
77
77
|
eventHandler: BinLogEventHandler;
|
|
78
78
|
sourceTables: TablePattern[];
|
|
79
|
+
/**
|
|
80
|
+
* Id that identifies this replication client.
|
|
81
|
+
*/
|
|
79
82
|
serverId: number;
|
|
83
|
+
/**
|
|
84
|
+
* The server uuid of the source MySQL server that is being replicated.
|
|
85
|
+
*/
|
|
86
|
+
activeServerUuid: string;
|
|
80
87
|
startGTID: common.ReplicatedGTID;
|
|
81
88
|
logger?: Logger;
|
|
82
89
|
keepAliveInactivitySeconds?: number;
|
|
@@ -88,8 +95,6 @@ export interface BinLogListenerOptions {
|
|
|
88
95
|
*/
|
|
89
96
|
export class BinLogListener {
|
|
90
97
|
private sqlParser: ParserType;
|
|
91
|
-
private connectionManager: MySQLConnectionManager;
|
|
92
|
-
private eventHandler: BinLogEventHandler;
|
|
93
98
|
private binLogPosition: common.BinLogPosition;
|
|
94
99
|
private currentGTID: common.ReplicatedGTID;
|
|
95
100
|
private logger: Logger;
|
|
@@ -101,6 +106,7 @@ export class BinLogListener {
|
|
|
101
106
|
|
|
102
107
|
// Flag to indicate if are currently in a transaction that involves multiple row mutation events.
|
|
103
108
|
private isTransactionOpen = false;
|
|
109
|
+
|
|
104
110
|
zongji: ZongJi;
|
|
105
111
|
processingQueue: async.QueueObject<BinLogEvent>;
|
|
106
112
|
|
|
@@ -111,9 +117,8 @@ export class BinLogListener {
|
|
|
111
117
|
|
|
112
118
|
constructor(public options: BinLogListenerOptions) {
|
|
113
119
|
this.logger = options.logger ?? defaultLogger;
|
|
114
|
-
|
|
115
|
-
this.
|
|
116
|
-
this.binLogPosition = options.startGTID.position;
|
|
120
|
+
// Copy the position: the listener mutates it as events are processed, and the caller's startGTID must not change
|
|
121
|
+
this.binLogPosition = { ...options.startGTID.position };
|
|
117
122
|
this.currentGTID = options.startGTID;
|
|
118
123
|
this.sqlParser = new Parser();
|
|
119
124
|
this.processingQueue = this.createProcessingQueue();
|
|
@@ -122,6 +127,18 @@ export class BinLogListener {
|
|
|
122
127
|
this.databaseFilter = this.createDatabaseFilter(options.sourceTables);
|
|
123
128
|
}
|
|
124
129
|
|
|
130
|
+
private get connectionManager(): MySQLConnectionManager {
|
|
131
|
+
return this.options.connectionManager;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private get eventHandler(): BinLogEventHandler {
|
|
135
|
+
return this.options.eventHandler;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private get activeServerUuid(): string {
|
|
139
|
+
return this.options.activeServerUuid;
|
|
140
|
+
}
|
|
141
|
+
|
|
125
142
|
/**
|
|
126
143
|
* The queue memory limit in bytes as defined in the connection options.
|
|
127
144
|
* @private
|
|
@@ -292,29 +309,41 @@ export class BinLogListener {
|
|
|
292
309
|
return async (evt: BinLogEvent) => {
|
|
293
310
|
switch (true) {
|
|
294
311
|
case zongji_utils.eventIsGTIDLog(evt):
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
312
|
+
const transactionGTID = common.ReplicatedGTID.fromBinLogEvent({
|
|
313
|
+
rawGtid: {
|
|
314
|
+
serverUuid: evt.serverId, // The server uuid this transaction originated from
|
|
315
|
+
transactionId: evt.transactionRange
|
|
299
316
|
},
|
|
300
317
|
position: {
|
|
301
318
|
filename: this.binLogPosition.filename,
|
|
302
319
|
offset: evt.nextPosition
|
|
303
320
|
}
|
|
304
321
|
});
|
|
322
|
+
|
|
323
|
+
if (transactionGTID.serverUuid !== this.activeServerUuid) {
|
|
324
|
+
throw new ReplicationAssertionError(
|
|
325
|
+
`Detected a transaction from a different MySQL server UUID: ${transactionGTID.serverUuid} than the server that is currently being replicated from: ${this.activeServerUuid}. ` +
|
|
326
|
+
`A re-snapshot is required to ensure consistency.`
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
this.currentGTID = transactionGTID;
|
|
305
331
|
this.binLogPosition.offset = evt.nextPosition;
|
|
332
|
+
|
|
306
333
|
await this.eventHandler.onTransactionStart({ timestamp: new Date(evt.timestamp) });
|
|
307
334
|
this.logger.info(`Processed GTID event: ${this.currentGTID.comparable}`);
|
|
308
335
|
break;
|
|
309
336
|
case zongji_utils.eventIsRotation(evt):
|
|
310
337
|
// The first event when starting replication is a synthetic Rotate event
|
|
311
|
-
// It describes the
|
|
338
|
+
// It describes the the position and file that the replica requested to start from
|
|
339
|
+
const isNewFile = this.binLogPosition.filename !== evt.binlogName;
|
|
340
|
+
|
|
312
341
|
this.binLogPosition.filename = evt.binlogName;
|
|
313
|
-
this.binLogPosition.offset = evt.
|
|
342
|
+
this.binLogPosition.offset = evt.position;
|
|
343
|
+
|
|
314
344
|
await this.eventHandler.onRotate();
|
|
315
345
|
|
|
316
|
-
|
|
317
|
-
if (newFile) {
|
|
346
|
+
if (isNewFile) {
|
|
318
347
|
this.logger.info(
|
|
319
348
|
`Processed Rotate event. New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`
|
|
320
349
|
);
|
|
@@ -359,11 +388,7 @@ export class BinLogListener {
|
|
|
359
388
|
break;
|
|
360
389
|
case zongji_utils.eventIsXid(evt):
|
|
361
390
|
this.isTransactionOpen = false;
|
|
362
|
-
|
|
363
|
-
const LSN = new common.ReplicatedGTID({
|
|
364
|
-
raw_gtid: this.currentGTID.raw,
|
|
365
|
-
position: this.binLogPosition
|
|
366
|
-
}).comparable;
|
|
391
|
+
const LSN = this.advanceCommitPosition(evt.nextPosition);
|
|
367
392
|
await this.eventHandler.onCommit(LSN);
|
|
368
393
|
this.logger.info(`Processed Xid event - transaction complete. LSN: ${LSN}.`);
|
|
369
394
|
break;
|
|
@@ -376,6 +401,21 @@ export class BinLogListener {
|
|
|
376
401
|
};
|
|
377
402
|
}
|
|
378
403
|
|
|
404
|
+
/**
|
|
405
|
+
* Advances the binlog position to the end of a committed transaction and updates the currentGTID to match.
|
|
406
|
+
* This ensures subsequent heartbeat keepalives report an LSN that is not behind the last commit LSN,
|
|
407
|
+
* which would otherwise block checkpoint creation until the next transaction arrives.
|
|
408
|
+
* Returns the commit LSN.
|
|
409
|
+
*/
|
|
410
|
+
private advanceCommitPosition(nextPosition: number): string {
|
|
411
|
+
this.binLogPosition.offset = nextPosition;
|
|
412
|
+
this.currentGTID = new common.ReplicatedGTID({
|
|
413
|
+
rawGtid: this.currentGTID.raw,
|
|
414
|
+
position: { ...this.binLogPosition }
|
|
415
|
+
});
|
|
416
|
+
return this.currentGTID.comparable;
|
|
417
|
+
}
|
|
418
|
+
|
|
379
419
|
private async processQueryEvent(event: BinLogQueryEvent): Promise<void> {
|
|
380
420
|
const { query, nextPosition } = event;
|
|
381
421
|
|
|
@@ -398,11 +438,7 @@ export class BinLogListener {
|
|
|
398
438
|
// DDL queries are auto commited, but do not come with a corresponding Xid event, in those cases we trigger a manual commit if we are not already in a transaction.
|
|
399
439
|
// Some DDL queries include row events, and in those cases will include a Xid event.
|
|
400
440
|
if (!this.isTransactionOpen) {
|
|
401
|
-
|
|
402
|
-
const LSN = new common.ReplicatedGTID({
|
|
403
|
-
raw_gtid: this.currentGTID.raw,
|
|
404
|
-
position: this.binLogPosition
|
|
405
|
-
}).comparable;
|
|
441
|
+
const LSN = this.advanceCommitPosition(nextPosition);
|
|
406
442
|
await this.eventHandler.onCommit(LSN);
|
|
407
443
|
}
|
|
408
444
|
|
|
@@ -419,11 +455,7 @@ export class BinLogListener {
|
|
|
419
455
|
await this.restartZongji();
|
|
420
456
|
}
|
|
421
457
|
} else if (!this.isTransactionOpen) {
|
|
422
|
-
|
|
423
|
-
const LSN = new common.ReplicatedGTID({
|
|
424
|
-
raw_gtid: this.currentGTID.raw,
|
|
425
|
-
position: this.binLogPosition
|
|
426
|
-
}).comparable;
|
|
458
|
+
const LSN = this.advanceCommitPosition(nextPosition);
|
|
427
459
|
await this.eventHandler.onCommit(LSN);
|
|
428
460
|
}
|
|
429
461
|
}
|
|
@@ -111,6 +111,24 @@ describe('BinlogListener tests', { timeout: 60_000 }, () => {
|
|
|
111
111
|
expect(eventHandler.lastKeepAlive).toEqual(binLogListener.options.startGTID.comparable);
|
|
112
112
|
});
|
|
113
113
|
|
|
114
|
+
test('Keepalive LSN after a commit is not less than the commit LSN', async () => {
|
|
115
|
+
binLogListener.options.keepAliveInactivitySeconds = 1;
|
|
116
|
+
await binLogListener.start();
|
|
117
|
+
|
|
118
|
+
await insertRows(connectionManager, 1);
|
|
119
|
+
await vi.waitFor(() => expect(eventHandler.commitCount).equals(1), { timeout: 5000 });
|
|
120
|
+
const commitLsn = eventHandler.lastCommitLsn!;
|
|
121
|
+
|
|
122
|
+
// Wait for a heartbeat keepalive that arrives after the commit.
|
|
123
|
+
// A keepalive LSN behind the commit LSN blocks checkpoint creation until the next transaction arrives.
|
|
124
|
+
await vi.waitFor(() => expect(eventHandler.lastKeepAlive && eventHandler.lastKeepAlive >= commitLsn).toBeTruthy(), {
|
|
125
|
+
timeout: 10_000
|
|
126
|
+
});
|
|
127
|
+
await binLogListener.stop();
|
|
128
|
+
// No binlog rotation happens in this test, so the keepalive LSN should exactly match the commit LSN
|
|
129
|
+
expect(eventHandler.lastKeepAlive).toEqual(commitLsn);
|
|
130
|
+
});
|
|
131
|
+
|
|
114
132
|
test('Schema change event: Rename table', async () => {
|
|
115
133
|
await binLogListener.start();
|
|
116
134
|
await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`);
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { ReplicatedGTID } from '@module/common/ReplicatedGTID.js';
|
|
2
|
+
import * as uuid from 'uuid';
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
4
|
+
|
|
5
|
+
describe('ReplicatedGTID', () => {
|
|
6
|
+
const SERVER_UUID = 'a7d0ff7b-0c0e-11f0-8b38-566fbaa00004';
|
|
7
|
+
const POSITION = { filename: 'binlog.000042', offset: 1234 };
|
|
8
|
+
|
|
9
|
+
describe('single GTID', () => {
|
|
10
|
+
test('exposes its raw value, server UUID, and binlog position', () => {
|
|
11
|
+
const gtid = new ReplicatedGTID({
|
|
12
|
+
rawGtid: `${SERVER_UUID}:5`,
|
|
13
|
+
position: POSITION
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
expect(gtid.raw).toEqual(`${SERVER_UUID}:5`);
|
|
17
|
+
expect(gtid.serverUuid).toEqual(SERVER_UUID);
|
|
18
|
+
expect(gtid.position).toEqual(POSITION);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test('formats a comparable LSN using the transaction id', () => {
|
|
22
|
+
const gtid = new ReplicatedGTID({
|
|
23
|
+
rawGtid: `${SERVER_UUID}:17`,
|
|
24
|
+
position: POSITION
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
|
|
28
|
+
expect(gtid.toString()).toEqual(gtid.comparable);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('normalizes surrounding whitespace', () => {
|
|
32
|
+
const gtid = new ReplicatedGTID({
|
|
33
|
+
rawGtid: ` \n\t${SERVER_UUID}:17 \r\n`,
|
|
34
|
+
position: POSITION
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
expect(gtid.raw).toEqual(`${SERVER_UUID}:17`);
|
|
38
|
+
expect(gtid.serverUuid).toEqual(SERVER_UUID);
|
|
39
|
+
expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('keeps the ZERO GTID format stable', () => {
|
|
43
|
+
expect(ReplicatedGTID.ZERO(SERVER_UUID).raw).toEqual(`${SERVER_UUID}:0`);
|
|
44
|
+
expect(ReplicatedGTID.ZERO(SERVER_UUID).comparable).toEqual(`0000000000000000|${SERVER_UUID}:0||0`);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe('validation', () => {
|
|
49
|
+
test.each([
|
|
50
|
+
['', 'missing server UUID and transaction id'],
|
|
51
|
+
[SERVER_UUID, 'missing transaction id'],
|
|
52
|
+
[`${SERVER_UUID}:`, 'empty transaction id'],
|
|
53
|
+
[`:${17}`, 'empty server UUID'],
|
|
54
|
+
[`${SERVER_UUID}:1-17`, 'transaction interval'],
|
|
55
|
+
[`${SERVER_UUID}:1:17`, 'multiple transaction components'],
|
|
56
|
+
[`${SERVER_UUID}:abc`, 'non-numeric transaction id'],
|
|
57
|
+
[`${SERVER_UUID}:-1`, 'negative transaction id'],
|
|
58
|
+
[`${SERVER_UUID}:17,another-server:9`, 'comma-separated GTID set'],
|
|
59
|
+
[`${SERVER_UUID}:17,\nanother-server:9`, 'newline-separated GTID set']
|
|
60
|
+
])('rejects %s (%s)', (rawGtid) => {
|
|
61
|
+
expect(() => new ReplicatedGTID({ rawGtid, position: POSITION })).toThrow();
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe('serialization', () => {
|
|
66
|
+
test('round-trips a single GTID', () => {
|
|
67
|
+
const gtid = new ReplicatedGTID({
|
|
68
|
+
rawGtid: `${SERVER_UUID}:17`,
|
|
69
|
+
position: POSITION
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
const deserialized = ReplicatedGTID.fromSerialized(gtid.comparable);
|
|
73
|
+
|
|
74
|
+
expect(deserialized.raw).toEqual(gtid.raw);
|
|
75
|
+
expect(deserialized.serverUuid).toEqual(SERVER_UUID);
|
|
76
|
+
expect(deserialized.position).toEqual(POSITION);
|
|
77
|
+
expect(deserialized.comparable).toEqual(gtid.comparable);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('throws on malformed serialized GTIDs', () => {
|
|
81
|
+
expect(() => ReplicatedGTID.fromSerialized('abc')).toThrow('Invalid serialized GTID');
|
|
82
|
+
expect(() => ReplicatedGTID.fromSerialized(`0000000000000001|${SERVER_UUID}:1|binlog.000001`)).toThrow(
|
|
83
|
+
'Invalid serialized GTID'
|
|
84
|
+
);
|
|
85
|
+
expect(() => ReplicatedGTID.fromSerialized(`0000000000000001|${SERVER_UUID}:1|binlog.000001|notanumber`)).toThrow(
|
|
86
|
+
'Invalid BinLog offset'
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('rejects a serialized GTID set', () => {
|
|
91
|
+
const serialized = `0000000000000017|${SERVER_UUID}:1-17|binlog.000042|1234`;
|
|
92
|
+
|
|
93
|
+
expect(() => ReplicatedGTID.fromSerialized(serialized)).toThrow('Expected a single transaction id');
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
describe('binlog events', () => {
|
|
98
|
+
test('creates a single GTID from a binlog event', () => {
|
|
99
|
+
const gtid = ReplicatedGTID.fromBinLogEvent({
|
|
100
|
+
rawGtid: {
|
|
101
|
+
serverUuid: Buffer.from(uuid.parse(SERVER_UUID)),
|
|
102
|
+
transactionId: 17
|
|
103
|
+
},
|
|
104
|
+
position: POSITION
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
expect(gtid.raw).toEqual(`${SERVER_UUID}:17`);
|
|
108
|
+
expect(gtid.position).toEqual(POSITION);
|
|
109
|
+
expect(gtid.comparable).toEqual(`0000000000000017|${SERVER_UUID}:17|binlog.000042|1234`);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
describe('LSN ordering', () => {
|
|
114
|
+
test('orders GTIDs from the same server by transaction id', () => {
|
|
115
|
+
const earlier = new ReplicatedGTID({ rawGtid: `${SERVER_UUID}:9`, position: POSITION });
|
|
116
|
+
const later = new ReplicatedGTID({ rawGtid: `${SERVER_UUID}:18`, position: POSITION });
|
|
117
|
+
|
|
118
|
+
expect(earlier.comparable < later.comparable).toBeTruthy();
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('orders LSNs for the same transaction by binlog offset', () => {
|
|
122
|
+
// The binlog offset is not zero-padded, so lexicographic ordering only holds for
|
|
123
|
+
// offsets with the same number of digits. This format cannot change while existing
|
|
124
|
+
// LSNs remain persisted in bucket storage.
|
|
125
|
+
const rawGtid = `${SERVER_UUID}:18`;
|
|
126
|
+
const transactionStart = new ReplicatedGTID({
|
|
127
|
+
rawGtid,
|
|
128
|
+
position: { filename: 'binlog.000042', offset: 157 }
|
|
129
|
+
});
|
|
130
|
+
const transactionEnd = new ReplicatedGTID({
|
|
131
|
+
rawGtid,
|
|
132
|
+
position: { filename: 'binlog.000042', offset: 300 }
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
expect(transactionStart.comparable < transactionEnd.comparable).toBeTruthy();
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { checkSourceConfiguration } from '@module/common/check-source-configuration.js';
|
|
2
|
+
import { describe, expect, test } from 'vitest';
|
|
3
|
+
import { createMockMySQLConnection } from './util.js';
|
|
4
|
+
|
|
5
|
+
describe('checkSourceConfiguration', () => {
|
|
6
|
+
test('accepts a primary MySQL server', async () => {
|
|
7
|
+
const { connection, query } = createConnection({ version: '8.4.0', replicaStatuses: [] });
|
|
8
|
+
|
|
9
|
+
await expect(checkSourceConfiguration(connection)).resolves.toEqual([]);
|
|
10
|
+
expect(query).toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
test('rejects a replica on MySQL 8.0.22 and later', async () => {
|
|
14
|
+
const { connection, query } = createConnection({
|
|
15
|
+
version: '8.0.22',
|
|
16
|
+
replicaStatuses: [{ Channel_Name: '' }]
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
await expect(checkSourceConfiguration(connection)).resolves.toContain(
|
|
20
|
+
'Connecting PowerSync to a MySQL replica is not supported. Please connect PowerSync directly to the primary server.'
|
|
21
|
+
);
|
|
22
|
+
expect(query).toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
|
|
23
|
+
expect(query).not.toHaveBeenCalledWith('SHOW SLAVE STATUS', []);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test('uses legacy replica-status syntax before MySQL 8.0.22', async () => {
|
|
27
|
+
const { connection, query } = createConnection({
|
|
28
|
+
version: '5.7.44',
|
|
29
|
+
replicaStatuses: [{ Channel_Name: '' }]
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
await expect(checkSourceConfiguration(connection)).resolves.toContain(
|
|
33
|
+
'Connecting PowerSync to a MySQL replica is not supported. Please connect PowerSync directly to the primary server.'
|
|
34
|
+
);
|
|
35
|
+
expect(query).toHaveBeenCalledWith('SHOW SLAVE STATUS', []);
|
|
36
|
+
expect(query).not.toHaveBeenCalledWith('SHOW REPLICA STATUS', []);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
function createConnection(options: { version: string; replicaStatuses: Record<string, unknown>[] }) {
|
|
40
|
+
return createMockMySQLConnection(async (sql) => {
|
|
41
|
+
switch (sql.trim()) {
|
|
42
|
+
case 'SELECT VERSION() as version':
|
|
43
|
+
return [[{ version: options.version }], []];
|
|
44
|
+
case 'SHOW REPLICA STATUS':
|
|
45
|
+
case 'SHOW SLAVE STATUS':
|
|
46
|
+
return [options.replicaStatuses, []];
|
|
47
|
+
case "SHOW VARIABLES LIKE 'binlog_format';":
|
|
48
|
+
return [[{ Value: 'ROW' }], []];
|
|
49
|
+
case "SHOW GLOBAL VARIABLES LIKE 'binlog_row_image';":
|
|
50
|
+
return [[{ Value: 'FULL' }], []];
|
|
51
|
+
default:
|
|
52
|
+
if (sql.includes('@@GLOBAL.gtid_mode AS gtid_mode')) {
|
|
53
|
+
return [
|
|
54
|
+
[
|
|
55
|
+
{
|
|
56
|
+
gtid_mode: 'ON',
|
|
57
|
+
log_bin: 1,
|
|
58
|
+
server_id: 1,
|
|
59
|
+
binlog_file: '/var/lib/mysql/binlog',
|
|
60
|
+
binlog_index_file: '/var/lib/mysql/binlog.index'
|
|
61
|
+
}
|
|
62
|
+
],
|
|
63
|
+
[]
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
throw new Error(`Unexpected query: ${sql}`);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
});
|