@vlasky/zongji 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +30 -5
- package/example.js +5 -0
- package/index.d.ts +84 -2
- package/index.js +167 -59
- package/lib/binlog_event.js +39 -0
- package/lib/code_map.js +9 -1
- package/lib/common.js +71 -43
- package/lib/json_decode.js +61 -18
- package/lib/packet/binlog.js +7 -0
- package/lib/reader.js +4 -143
- package/lib/sequence/binlog.js +29 -7
- package/package.json +8 -5
- package/.travis.yml +0 -15
- package/docker-compose.yml +0 -33
- package/docker-test.sh +0 -11
- package/eslint.config.js +0 -35
- package/jsconfig.json +0 -15
- package/lib/packet/combinlog.js +0 -27
- package/lib/packet/index.js +0 -66
- package/scripts/start-mysql.sh +0 -28
package/index.js
CHANGED
|
@@ -7,23 +7,18 @@ const ConnectionConfigMap = {
|
|
|
7
7
|
'Pool': obj => obj.config.connectionConfig,
|
|
8
8
|
};
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const TableInfoQuery = `SELECT
|
|
11
11
|
COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME,
|
|
12
12
|
COLUMN_COMMENT, COLUMN_TYPE
|
|
13
|
-
FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA
|
|
13
|
+
FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=?
|
|
14
14
|
ORDER BY ORDINAL_POSITION`;
|
|
15
15
|
|
|
16
|
-
// Simple format function to replace %s placeholders
|
|
17
|
-
function formatSql(template, ...args) {
|
|
18
|
-
let i = 0;
|
|
19
|
-
return template.replace(/%s/g, () => args[i++]);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
16
|
class ZongJi extends EventEmitter {
|
|
23
17
|
constructor(dsn) {
|
|
24
18
|
super();
|
|
25
19
|
|
|
26
20
|
this._pendingErrors = [];
|
|
21
|
+
this._pendingErrorTimer = null;
|
|
27
22
|
this.on('newListener', (event) => {
|
|
28
23
|
if (event !== 'error' || this._pendingErrors.length === 0) {
|
|
29
24
|
return;
|
|
@@ -39,8 +34,12 @@ class ZongJi extends EventEmitter {
|
|
|
39
34
|
this._filters({});
|
|
40
35
|
this.ctrlCallbacks = [];
|
|
41
36
|
this.tableMap = {};
|
|
37
|
+
this._warnedUnsupported = new Set();
|
|
38
|
+
this._currentGtid = undefined;
|
|
42
39
|
this.ready = false;
|
|
43
40
|
this.stopped = false;
|
|
41
|
+
this._starting = false;
|
|
42
|
+
this._startEpoch = 0;
|
|
44
43
|
this.useChecksum = false;
|
|
45
44
|
|
|
46
45
|
this._dsn = dsn;
|
|
@@ -54,7 +53,12 @@ class ZongJi extends EventEmitter {
|
|
|
54
53
|
const createConnection = (options) => {
|
|
55
54
|
const emitError = (err) => {
|
|
56
55
|
if (this.listenerCount('error') === 0) {
|
|
56
|
+
// Buffer errors that fire before the caller attaches a listener
|
|
57
|
+
// (typically within the same tick as construction/start). If no
|
|
58
|
+
// listener ever appears, fall back to EventEmitter's default
|
|
59
|
+
// behaviour (throw) rather than swallowing failures silently.
|
|
57
60
|
this._pendingErrors.push(err);
|
|
61
|
+
this._schedulePendingErrorCheck();
|
|
58
62
|
return;
|
|
59
63
|
}
|
|
60
64
|
this.emit('error', err);
|
|
@@ -95,6 +99,23 @@ class ZongJi extends EventEmitter {
|
|
|
95
99
|
this.connection = createConnection(binlogDsn);
|
|
96
100
|
}
|
|
97
101
|
|
|
102
|
+
_schedulePendingErrorCheck() {
|
|
103
|
+
if (this._pendingErrorTimer) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
this._pendingErrorTimer = setImmediate(() => {
|
|
107
|
+
this._pendingErrorTimer = null;
|
|
108
|
+
if (this.listenerCount('error') > 0 || this._pendingErrors.length === 0) {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const pending = this._pendingErrors.slice();
|
|
112
|
+
this._pendingErrors.length = 0;
|
|
113
|
+
// No 'error' listener was attached within a macrotask; re-emit so
|
|
114
|
+
// Node's unhandled 'error' semantics apply (throws)
|
|
115
|
+
pending.forEach(err => this.emit('error', err));
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
98
119
|
_isChecksumEnabled(next) {
|
|
99
120
|
const SelectChecksumParamSql = 'select @@GLOBAL.binlog_checksum as checksum';
|
|
100
121
|
const SetChecksumSql = 'set @master_binlog_checksum=@@global.binlog_checksum';
|
|
@@ -184,19 +205,28 @@ class ZongJi extends EventEmitter {
|
|
|
184
205
|
}
|
|
185
206
|
|
|
186
207
|
_fetchTableInfo(tableMapEvent, next) {
|
|
187
|
-
const sql = formatSql(TableInfoQueryTemplate,
|
|
188
|
-
tableMapEvent.schemaName, tableMapEvent.tableName);
|
|
189
|
-
|
|
190
208
|
if (!this.ctrlConnection ||
|
|
191
209
|
this.ctrlConnection.state === 'disconnected' ||
|
|
192
210
|
this.ctrlConnection._fatalError ||
|
|
193
211
|
this.ctrlConnection._protocolError ||
|
|
194
212
|
this.ctrlConnection._closing) {
|
|
213
|
+
// The binlog connection stays paused, so processing has halted.
|
|
214
|
+
// During stop() that is expected; otherwise it must not be silent.
|
|
215
|
+
if (!this.stopped) {
|
|
216
|
+
this.emit('error', new Error(
|
|
217
|
+
'Control connection unavailable while fetching column metadata ' +
|
|
218
|
+
'for ' + tableMapEvent.schemaName + '.' + tableMapEvent.tableName +
|
|
219
|
+
'. Binlog processing has halted; call stop() then start() to resume.'));
|
|
220
|
+
}
|
|
195
221
|
return;
|
|
196
222
|
}
|
|
197
223
|
|
|
224
|
+
const params = [tableMapEvent.schemaName, tableMapEvent.tableName];
|
|
198
225
|
try {
|
|
199
|
-
|
|
226
|
+
// execute() uses a server-side prepared statement: parameters are sent
|
|
227
|
+
// out-of-band (never spliced into SQL text) and the statement is cached
|
|
228
|
+
// per connection, so repeated metadata lookups avoid re-parsing.
|
|
229
|
+
this.ctrlConnection.execute(TableInfoQuery, params, (err, rows) => {
|
|
200
230
|
if (err) {
|
|
201
231
|
// Errors should be emitted
|
|
202
232
|
this.emit('error', err);
|
|
@@ -234,12 +264,14 @@ class ZongJi extends EventEmitter {
|
|
|
234
264
|
filename,
|
|
235
265
|
position,
|
|
236
266
|
startAtEnd,
|
|
267
|
+
nonBlock,
|
|
237
268
|
} = {}) {
|
|
238
269
|
this.options = {
|
|
239
270
|
serverId,
|
|
240
271
|
filename,
|
|
241
272
|
position,
|
|
242
273
|
startAtEnd,
|
|
274
|
+
nonBlock,
|
|
243
275
|
};
|
|
244
276
|
}
|
|
245
277
|
|
|
@@ -257,6 +289,28 @@ class ZongJi extends EventEmitter {
|
|
|
257
289
|
includeSchema,
|
|
258
290
|
excludeSchema,
|
|
259
291
|
};
|
|
292
|
+
|
|
293
|
+
// Precompiled lookups so per-event filtering is O(1). Only own keys of
|
|
294
|
+
// the schema objects are considered (no prototype leakage).
|
|
295
|
+
const compileSchemaFilter = (schema) => {
|
|
296
|
+
if (schema === undefined || schema === null) {
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
const compiled = new Map();
|
|
300
|
+
for (const database of Object.keys(schema)) {
|
|
301
|
+
const tables = schema[database];
|
|
302
|
+
compiled.set(database, tables === true ?
|
|
303
|
+
true : new Set(Array.isArray(tables) ? tables : []));
|
|
304
|
+
}
|
|
305
|
+
return compiled;
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
this._includeEventsSet = includeEvents === undefined ?
|
|
309
|
+
undefined : new Set(Array.isArray(includeEvents) ? includeEvents : []);
|
|
310
|
+
this._excludeEventsSet =
|
|
311
|
+
new Set(Array.isArray(excludeEvents) ? excludeEvents : []);
|
|
312
|
+
this._includeSchemaMap = compileSchemaFilter(includeSchema);
|
|
313
|
+
this._excludeSchemaMap = compileSchemaFilter(excludeSchema) || new Map();
|
|
260
314
|
}
|
|
261
315
|
|
|
262
316
|
get(name) {
|
|
@@ -289,6 +343,22 @@ class ZongJi extends EventEmitter {
|
|
|
289
343
|
return;
|
|
290
344
|
}
|
|
291
345
|
|
|
346
|
+
// A duplicate start() while one is already initialising is ignored;
|
|
347
|
+
// the first call completes. But if stop() intervened, this start()
|
|
348
|
+
// must proceed as a restart: the epoch below makes the stale
|
|
349
|
+
// initialisation chain abort, so exactly one binlog dump command is
|
|
350
|
+
// ever enqueued.
|
|
351
|
+
if (this._starting && !this.stopped) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
this._starting = true;
|
|
355
|
+
this._startEpoch += 1;
|
|
356
|
+
const epoch = this._startEpoch;
|
|
357
|
+
|
|
358
|
+
// A resumed stream must not attribute early events to a GTID seen
|
|
359
|
+
// before the restart
|
|
360
|
+
this._currentGtid = undefined;
|
|
361
|
+
|
|
292
362
|
this.stopped = false;
|
|
293
363
|
|
|
294
364
|
if (!this.connection || !this.ctrlConnection) {
|
|
@@ -330,6 +400,16 @@ class ZongJi extends EventEmitter {
|
|
|
330
400
|
});
|
|
331
401
|
};
|
|
332
402
|
|
|
403
|
+
// Attach the current transaction's GTID (tracked at the packet layer,
|
|
404
|
+
// even when 'gtid' events are filtered out). Gtid/AnonymousGtid events
|
|
405
|
+
// keep their own parsed value.
|
|
406
|
+
const attachGtid = (event) => {
|
|
407
|
+
if (!('gtid' in event)) {
|
|
408
|
+
event.gtid = this._currentGtid;
|
|
409
|
+
}
|
|
410
|
+
return event;
|
|
411
|
+
};
|
|
412
|
+
|
|
333
413
|
const binlogHandler = (error, event) => {
|
|
334
414
|
if (error) {
|
|
335
415
|
return this.emit('error', error);
|
|
@@ -347,6 +427,7 @@ class ZongJi extends EventEmitter {
|
|
|
347
427
|
if (!tableMap || tableMap.tableName !== event.tableName || tableMap.columns.length !== event.columnCount) {
|
|
348
428
|
if (!this.connection) return;
|
|
349
429
|
this.connection.pause();
|
|
430
|
+
attachGtid(event);
|
|
350
431
|
this._fetchTableInfo(event, () => {
|
|
351
432
|
// merge the column info with metadata
|
|
352
433
|
event.updateColumnInfo();
|
|
@@ -364,7 +445,7 @@ class ZongJi extends EventEmitter {
|
|
|
364
445
|
break;
|
|
365
446
|
}
|
|
366
447
|
this.options.position = event.nextPosition;
|
|
367
|
-
this.emit('binlog', event);
|
|
448
|
+
this.emit('binlog', attachGtid(event));
|
|
368
449
|
};
|
|
369
450
|
|
|
370
451
|
let promises = [new Promise(testChecksum)];
|
|
@@ -375,7 +456,13 @@ class ZongJi extends EventEmitter {
|
|
|
375
456
|
|
|
376
457
|
Promise.all(promises)
|
|
377
458
|
.then(() => {
|
|
378
|
-
//
|
|
459
|
+
// Abort if a newer start() superseded this one (after a stop())
|
|
460
|
+
// while promises were pending
|
|
461
|
+
if (epoch !== this._startEpoch) {
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
this._starting = false;
|
|
465
|
+
// Abort if stop() was called while promises were pending
|
|
379
466
|
if (this.stopped) {
|
|
380
467
|
return;
|
|
381
468
|
}
|
|
@@ -384,8 +471,9 @@ class ZongJi extends EventEmitter {
|
|
|
384
471
|
this.ready = true;
|
|
385
472
|
this.emit('ready');
|
|
386
473
|
|
|
387
|
-
// Final check right before addCommand - connection may have been
|
|
388
|
-
|
|
474
|
+
// Final check right before addCommand - connection may have been
|
|
475
|
+
// destroyed or this start() superseded by a ready handler
|
|
476
|
+
if (this.stopped || epoch !== this._startEpoch) {
|
|
389
477
|
return;
|
|
390
478
|
}
|
|
391
479
|
|
|
@@ -413,6 +501,10 @@ class ZongJi extends EventEmitter {
|
|
|
413
501
|
this.connection.addCommand(new this.BinlogClass(binlogHandler));
|
|
414
502
|
})
|
|
415
503
|
.catch(err => {
|
|
504
|
+
if (epoch !== this._startEpoch) {
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
this._starting = false;
|
|
416
508
|
this.emit('error', err);
|
|
417
509
|
});
|
|
418
510
|
|
|
@@ -427,10 +519,25 @@ class ZongJi extends EventEmitter {
|
|
|
427
519
|
return;
|
|
428
520
|
}
|
|
429
521
|
|
|
522
|
+
// Errors emitted by a connection we are deliberately destroying are
|
|
523
|
+
// teardown noise (e.g. in-flight queries failing with
|
|
524
|
+
// ERR_STREAM_WRITE_AFTER_END); do not forward them to the caller
|
|
525
|
+
const silenceErrors = (conn) => {
|
|
526
|
+
conn.removeAllListeners('error');
|
|
527
|
+
conn.on('error', () => {});
|
|
528
|
+
};
|
|
529
|
+
|
|
530
|
+
// Capture the connections this stop() owns: finish() may fire from an
|
|
531
|
+
// async callback after a subsequent start() has already created
|
|
532
|
+
// replacement connections, and must not touch those
|
|
533
|
+
const ctrlConnection = this.ctrlConnection;
|
|
534
|
+
const ctrlToClose = this.ctrlConnectionOwner ? ctrlConnection : null;
|
|
535
|
+
|
|
430
536
|
// Binary log connection does not end with destroy()
|
|
431
537
|
let connectionThreadId = null;
|
|
432
538
|
if (this.connection) {
|
|
433
539
|
connectionThreadId = this.connection.threadId;
|
|
540
|
+
silenceErrors(this.connection);
|
|
434
541
|
this.connection.destroy();
|
|
435
542
|
// @ts-ignore - internal mysql2 API
|
|
436
543
|
if (this.connection.stream && typeof this.connection.stream.unref === 'function') {
|
|
@@ -443,24 +550,29 @@ class ZongJi extends EventEmitter {
|
|
|
443
550
|
const finish = () => {
|
|
444
551
|
if (finished) return;
|
|
445
552
|
finished = true;
|
|
446
|
-
if (
|
|
447
|
-
|
|
553
|
+
if (ctrlToClose) {
|
|
554
|
+
silenceErrors(ctrlToClose);
|
|
555
|
+
ctrlToClose.destroy();
|
|
448
556
|
// @ts-ignore - internal mysql2 API
|
|
449
|
-
if (
|
|
557
|
+
if (ctrlToClose.stream && typeof ctrlToClose.stream.unref === 'function') {
|
|
450
558
|
// @ts-ignore - internal mysql2 API
|
|
451
|
-
|
|
559
|
+
ctrlToClose.stream.unref();
|
|
560
|
+
}
|
|
561
|
+
if (this.ctrlConnection === ctrlToClose) {
|
|
562
|
+
this.ctrlConnection = null;
|
|
452
563
|
}
|
|
453
|
-
this.ctrlConnection = null;
|
|
454
564
|
}
|
|
455
565
|
this.emit('stopped');
|
|
456
566
|
};
|
|
457
567
|
|
|
458
|
-
if (!
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
this.ctrlConnection
|
|
568
|
+
if (!ctrlConnection ||
|
|
569
|
+
ctrlConnection.state === 'disconnected' ||
|
|
570
|
+
ctrlConnection._fatalError ||
|
|
571
|
+
ctrlConnection._protocolError ||
|
|
572
|
+
ctrlConnection._closing) {
|
|
573
|
+
if (this.ctrlConnection === ctrlConnection) {
|
|
574
|
+
this.ctrlConnection = null;
|
|
575
|
+
}
|
|
464
576
|
return finish();
|
|
465
577
|
}
|
|
466
578
|
|
|
@@ -470,7 +582,7 @@ class ZongJi extends EventEmitter {
|
|
|
470
582
|
|
|
471
583
|
const killTimeout = setTimeout(finish, 1000);
|
|
472
584
|
try {
|
|
473
|
-
|
|
585
|
+
ctrlConnection.query(
|
|
474
586
|
'KILL ' + connectionThreadId,
|
|
475
587
|
() => {
|
|
476
588
|
clearTimeout(killTimeout);
|
|
@@ -483,44 +595,40 @@ class ZongJi extends EventEmitter {
|
|
|
483
595
|
}
|
|
484
596
|
}
|
|
485
597
|
|
|
598
|
+
// Emit an error the first time an undecodable event type arrives so that
|
|
599
|
+
// dropped row changes (e.g. compressed transactions) are never silent.
|
|
600
|
+
_warnUnsupportedEvent(EventClass) {
|
|
601
|
+
if (this._warnedUnsupported.has(EventClass.name)) {
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
this._warnedUnsupported.add(EventClass.name);
|
|
605
|
+
this.emit('error', new Error(EventClass.unsupportedReason));
|
|
606
|
+
}
|
|
607
|
+
|
|
486
608
|
// It includes every events by default.
|
|
487
609
|
_skipEvent(name) {
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
let excluded = Array.isArray(excludes) && (excludes.indexOf(name) > -1);
|
|
494
|
-
|
|
495
|
-
return excluded || !included;
|
|
610
|
+
if (this._excludeEventsSet.has(name)) {
|
|
611
|
+
return true;
|
|
612
|
+
}
|
|
613
|
+
return this._includeEventsSet !== undefined &&
|
|
614
|
+
!this._includeEventsSet.has(name);
|
|
496
615
|
}
|
|
497
616
|
|
|
498
617
|
// It doesn't skip any schema by default.
|
|
499
618
|
_skipSchema(database, table) {
|
|
500
|
-
const
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
(database in includes) &&
|
|
506
|
-
(
|
|
507
|
-
includes[database] === true ||
|
|
508
|
-
(
|
|
509
|
-
Array.isArray(includes[database]) &&
|
|
510
|
-
includes[database].indexOf(table) > -1
|
|
511
|
-
)
|
|
512
|
-
)
|
|
513
|
-
);
|
|
514
|
-
let excluded = (database in excludes) &&
|
|
515
|
-
(
|
|
516
|
-
excludes[database] === true ||
|
|
517
|
-
(
|
|
518
|
-
Array.isArray(excludes[database]) &&
|
|
519
|
-
excludes[database].indexOf(table) > -1
|
|
520
|
-
)
|
|
521
|
-
);
|
|
619
|
+
const excludeEntry = this._excludeSchemaMap.get(database);
|
|
620
|
+
if (excludeEntry === true ||
|
|
621
|
+
(excludeEntry instanceof Set && excludeEntry.has(table))) {
|
|
622
|
+
return true;
|
|
623
|
+
}
|
|
522
624
|
|
|
523
|
-
|
|
625
|
+
if (this._includeSchemaMap === undefined) {
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
const includeEntry = this._includeSchemaMap.get(database);
|
|
629
|
+
const included = includeEntry === true ||
|
|
630
|
+
(includeEntry instanceof Set && includeEntry.has(table));
|
|
631
|
+
return !included;
|
|
524
632
|
}
|
|
525
633
|
}
|
|
526
634
|
|
package/lib/binlog_event.js
CHANGED
|
@@ -364,6 +364,43 @@ class Unknown extends BinlogEvent {
|
|
|
364
364
|
}
|
|
365
365
|
}
|
|
366
366
|
|
|
367
|
+
/* MySQL 8.0.20+ wraps whole transactions in a single compressed event when
|
|
368
|
+
* binlog_transaction_compression=ON. The embedded row events are
|
|
369
|
+
* zstd-compressed and zongji cannot decode them, so the row changes they
|
|
370
|
+
* carry would be silently lost. The first occurrence emits an error via
|
|
371
|
+
* ZongJi#_warnUnsupportedEvent (see unsupportedReason below).
|
|
372
|
+
*/
|
|
373
|
+
class TransactionPayload extends BinlogEvent {
|
|
374
|
+
constructor(parser, options, zongji) {
|
|
375
|
+
super(parser, options);
|
|
376
|
+
skipEventRemainder(parser, zongji);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
static unsupportedReason =
|
|
380
|
+
'Server sent a TRANSACTION_PAYLOAD_EVENT (binlog_transaction_compression=ON). ' +
|
|
381
|
+
'zongji cannot decode compressed transactions, so their row events are NOT ' +
|
|
382
|
+
'being emitted. Set binlog_transaction_compression=OFF on the server to ' +
|
|
383
|
+
'receive these changes.';
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/* MySQL 8.0+ emits partial JSON row updates when
|
|
387
|
+
* binlog_row_value_options=PARTIAL_JSON. The JSON diff format is not
|
|
388
|
+
* decodable by zongji, so these updates would be silently lost. The first
|
|
389
|
+
* occurrence emits an error via ZongJi#_warnUnsupportedEvent.
|
|
390
|
+
*/
|
|
391
|
+
class PartialUpdateRows extends BinlogEvent {
|
|
392
|
+
constructor(parser, options, zongji) {
|
|
393
|
+
super(parser, options);
|
|
394
|
+
skipEventRemainder(parser, zongji);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
static unsupportedReason =
|
|
398
|
+
'Server sent a PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=PARTIAL_JSON). ' +
|
|
399
|
+
'zongji cannot decode partial JSON updates, so these UPDATE row events are ' +
|
|
400
|
+
'NOT being emitted. Clear binlog_row_value_options on the server to receive ' +
|
|
401
|
+
'these changes.';
|
|
402
|
+
}
|
|
403
|
+
|
|
367
404
|
export {
|
|
368
405
|
BinlogEvent,
|
|
369
406
|
Rotate,
|
|
@@ -375,5 +412,7 @@ export {
|
|
|
375
412
|
Gtid,
|
|
376
413
|
AnonymousGtid,
|
|
377
414
|
PreviousGtids,
|
|
415
|
+
TransactionPayload,
|
|
416
|
+
PartialUpdateRows,
|
|
378
417
|
Unknown
|
|
379
418
|
};
|
package/lib/code_map.js
CHANGED
|
@@ -37,7 +37,13 @@ const CodeEvent = [
|
|
|
37
37
|
'DELETE_ROWS_EVENT_V2',
|
|
38
38
|
'GTID_LOG_EVENT',
|
|
39
39
|
'ANONYMOUS_GTID_LOG_EVENT',
|
|
40
|
-
'PREVIOUS_GTIDS_LOG_EVENT'
|
|
40
|
+
'PREVIOUS_GTIDS_LOG_EVENT',
|
|
41
|
+
'TRANSACTION_CONTEXT_EVENT',
|
|
42
|
+
'VIEW_CHANGE_EVENT',
|
|
43
|
+
'XA_PREPARE_LOG_EVENT',
|
|
44
|
+
'PARTIAL_UPDATE_ROWS_EVENT',
|
|
45
|
+
'TRANSACTION_PAYLOAD_EVENT',
|
|
46
|
+
'HEARTBEAT_LOG_EVENT_V2'
|
|
41
47
|
];
|
|
42
48
|
|
|
43
49
|
const EventClass = {
|
|
@@ -50,6 +56,8 @@ const EventClass = {
|
|
|
50
56
|
GTID_LOG_EVENT: events.Gtid,
|
|
51
57
|
ANONYMOUS_GTID_LOG_EVENT: events.AnonymousGtid,
|
|
52
58
|
PREVIOUS_GTIDS_LOG_EVENT: events.PreviousGtids,
|
|
59
|
+
TRANSACTION_PAYLOAD_EVENT: events.TransactionPayload,
|
|
60
|
+
PARTIAL_UPDATE_ROWS_EVENT: events.PartialUpdateRows,
|
|
53
61
|
|
|
54
62
|
TABLE_MAP_EVENT: events.TableMap,
|
|
55
63
|
DELETE_ROWS_EVENT_V1: rowsEvents.DeleteRows,
|
package/lib/common.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import iconv from 'iconv-lite';
|
|
2
|
-
import decodeJson from './json_decode.js';
|
|
2
|
+
import decodeJson, { convertBigInts, jsonToText } from './json_decode.js';
|
|
3
3
|
import * as dtDecode from './datetime_decode.js';
|
|
4
|
-
import bigInt from 'big-integer';
|
|
5
4
|
|
|
6
5
|
export const MysqlTypes = {
|
|
7
6
|
DECIMAL: 0,
|
|
@@ -40,7 +39,27 @@ export const MysqlTypes = {
|
|
|
40
39
|
};
|
|
41
40
|
|
|
42
41
|
const TWO_TO_POWER_THIRTY_TWO = Math.pow(2, 32);
|
|
43
|
-
const TWO_TO_POWER_SIXTY_THREE =
|
|
42
|
+
const TWO_TO_POWER_SIXTY_THREE = 2n ** 63n;
|
|
43
|
+
const TWO_TO_POWER_SIXTY_FOUR = 2n ** 64n;
|
|
44
|
+
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
|
45
|
+
const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
|
|
46
|
+
|
|
47
|
+
// Combine two unsigned 32-bit halves into an exact unsigned 64-bit value.
|
|
48
|
+
// Returns a Number when at or below 2^53 - 1, otherwise an exact string.
|
|
49
|
+
export function unsignedInt64ToNumberOrString(high, low) {
|
|
50
|
+
const value = (BigInt(high) << 32n) | BigInt(low);
|
|
51
|
+
return value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Combine two unsigned 32-bit halves into an exact signed (two's
|
|
55
|
+
// complement) 64-bit value. Returns a Number when within the safe integer
|
|
56
|
+
// range, otherwise an exact string.
|
|
57
|
+
export function signedInt64ToNumberOrString(high, low) {
|
|
58
|
+
const value = BigInt.asIntN(64, (BigInt(high) << 32n) | BigInt(low));
|
|
59
|
+
return (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) ?
|
|
60
|
+
Number(value) : value.toString();
|
|
61
|
+
}
|
|
62
|
+
|
|
44
63
|
// This function will return a Number
|
|
45
64
|
// if the result < Math.MAX_SAFE_INTEGER or result > Math.MIN_SAFE_INTEGER,
|
|
46
65
|
// otherwise, will return a string.
|
|
@@ -48,11 +67,11 @@ export const parseUInt64 = function(parser) {
|
|
|
48
67
|
const low = parser.parseUnsignedNumber(4);
|
|
49
68
|
const high = parser.parseUnsignedNumber(4);
|
|
50
69
|
|
|
51
|
-
if (high >>> 21) { //
|
|
52
|
-
return
|
|
70
|
+
if (high >>> 21) { // value is 2^53 or more, exceeds safe Number range
|
|
71
|
+
return unsignedInt64ToNumberOrString(high, low);
|
|
53
72
|
}
|
|
54
73
|
|
|
55
|
-
return (high *
|
|
74
|
+
return (high * TWO_TO_POWER_THIRTY_TWO) + low;
|
|
56
75
|
};
|
|
57
76
|
|
|
58
77
|
export function parseUInt48(parser) {
|
|
@@ -115,12 +134,6 @@ export const parseIEEE754Float = function(high, low) {
|
|
|
115
134
|
}
|
|
116
135
|
};
|
|
117
136
|
|
|
118
|
-
export function getUInt32Value(input) {
|
|
119
|
-
// Last bit is not sign, it is part of value!
|
|
120
|
-
if (input & (1 << 31)) return Math.pow(2, 31) + (input & ((1 << 31) -1));
|
|
121
|
-
else return input;
|
|
122
|
-
};
|
|
123
|
-
|
|
124
137
|
const parseAnyInt = function(parser, column, columnSchema) {
|
|
125
138
|
let result, size;
|
|
126
139
|
switch (column.type) {
|
|
@@ -151,16 +164,13 @@ const parseAnyInt = function(parser, column, columnSchema) {
|
|
|
151
164
|
// Flip bits on negative signed integer
|
|
152
165
|
if (!int64 && (result & (1 << (length - 1)))) {
|
|
153
166
|
result = ((result ^ (Math.pow(2, length) - 1)) * -1) - 1;
|
|
154
|
-
} else if (int64
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
// Otherwise return a Number
|
|
162
|
-
else {
|
|
163
|
-
result = result.toJSNumber();
|
|
167
|
+
} else if (int64) {
|
|
168
|
+
const unsigned = BigInt(result);
|
|
169
|
+
if (unsigned >= TWO_TO_POWER_SIXTY_THREE) {
|
|
170
|
+
const signed = unsigned - TWO_TO_POWER_SIXTY_FOUR;
|
|
171
|
+
// Javascript Numbers only cover +-(2^53 - 1); return an exact
|
|
172
|
+
// string outside that range
|
|
173
|
+
result = signed >= MIN_SAFE_BIGINT ? Number(signed) : signed.toString();
|
|
164
174
|
}
|
|
165
175
|
}
|
|
166
176
|
}
|
|
@@ -185,7 +195,11 @@ const readIntBE = function(buf, offset, length, noAssert) {
|
|
|
185
195
|
// https://github.com/jeremycole/mysql_binlog/blob/master/lib/mysql_binlog/binlog_field_parser.rb
|
|
186
196
|
// Some more information about DECIMAL types:
|
|
187
197
|
// http://dev.mysql.com/doc/refman/5.5/en/precision-math-decimal-characteristics.html
|
|
188
|
-
|
|
198
|
+
// Returns the exact decimal value as a string (matching mysql2's default
|
|
199
|
+
// treatment of DECIMAL columns), or as a Number if the connection was
|
|
200
|
+
// configured with the mysql2 `decimalNumbers` option (which may lose
|
|
201
|
+
// precision beyond 15 significant digits).
|
|
202
|
+
const parseNewDecimal = function(parser, column, zongji) {
|
|
189
203
|
// Constants of format
|
|
190
204
|
const digitsPerInteger = 9;
|
|
191
205
|
const compressedBytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4];
|
|
@@ -197,38 +211,37 @@ const parseNewDecimal = function(parser, column) {
|
|
|
197
211
|
const compIntegral = integral - (uncompIntegral * digitsPerInteger);
|
|
198
212
|
const compFractional = scale - (uncompFractional * digitsPerInteger);
|
|
199
213
|
|
|
200
|
-
//
|
|
214
|
+
// Copy buffer portion: the sign bit is flipped in place below and the
|
|
215
|
+
// underlying packet buffer must not be mutated
|
|
201
216
|
const size = (uncompIntegral * 4) + compressedBytes[compIntegral] +
|
|
202
217
|
(uncompFractional * 4) + compressedBytes[compFractional];
|
|
203
|
-
const buffer =
|
|
218
|
+
const buffer = Buffer.from(
|
|
219
|
+
parser._buffer.subarray(parser._offset, parser._offset + size));
|
|
204
220
|
parser._offset += size; // Move binlog parser position forward
|
|
205
221
|
|
|
206
|
-
let str, mask;
|
|
207
222
|
let pos = 0;
|
|
208
223
|
const isPositive = (buffer.readUInt8(0) & (1 << 7)) === 128;
|
|
209
224
|
buffer.writeUInt8(buffer.readUInt8(0) ^ (1 << 7), 0);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
str = '';
|
|
213
|
-
mask = 0;
|
|
214
|
-
} else {
|
|
215
|
-
// Negative number
|
|
216
|
-
str = '-';
|
|
217
|
-
mask = -1;
|
|
218
|
-
}
|
|
225
|
+
// Negative numbers are stored with all bits flipped
|
|
226
|
+
const mask = isPositive ? 0 : -1;
|
|
219
227
|
|
|
220
228
|
// Build integer digits
|
|
229
|
+
let intDigits = '';
|
|
221
230
|
const compIntegralSize = compressedBytes[compIntegral];
|
|
222
231
|
if (compIntegralSize > 0) {
|
|
223
|
-
|
|
232
|
+
intDigits += (readIntBE(buffer, 0, compIntegralSize) ^ mask).toString(10);
|
|
224
233
|
pos += compIntegralSize;
|
|
225
234
|
}
|
|
226
235
|
|
|
227
236
|
for (let i = 0; i < uncompIntegral; i++) {
|
|
228
|
-
|
|
237
|
+
intDigits += zeroPad((buffer.readInt32BE(pos) ^ mask).toString(10), 9);
|
|
229
238
|
pos += 4;
|
|
230
239
|
}
|
|
231
240
|
|
|
241
|
+
// Strip leading zeros, keeping at least one digit
|
|
242
|
+
intDigits = intDigits.replace(/^0+(?=\d)/, '');
|
|
243
|
+
if (intDigits === '') intDigits = '0';
|
|
244
|
+
|
|
232
245
|
// Build fractional digits
|
|
233
246
|
let fractionDigits = '';
|
|
234
247
|
|
|
@@ -242,10 +255,13 @@ const parseNewDecimal = function(parser, column) {
|
|
|
242
255
|
fractionDigits += zeroPad((readIntBE(buffer, pos, compFractionalSize) ^ mask).toString(10), compFractional);
|
|
243
256
|
}
|
|
244
257
|
|
|
245
|
-
|
|
246
|
-
|
|
258
|
+
let str = (isPositive ? '' : '-') + intDigits;
|
|
259
|
+
if (scale > 0) {
|
|
260
|
+
str += '.' + fractionDigits;
|
|
261
|
+
}
|
|
247
262
|
|
|
248
|
-
|
|
263
|
+
const config = zongji && zongji.connection && zongji.connection.config;
|
|
264
|
+
return config && config.decimalNumbers ? parseFloat(str) : str;
|
|
249
265
|
};
|
|
250
266
|
|
|
251
267
|
// Did not work in place. Function cribbed from lines 311-363 of
|
|
@@ -380,7 +396,7 @@ export function readMysqlValue(
|
|
|
380
396
|
result = parseIEEE754Float(high, low);
|
|
381
397
|
break;
|
|
382
398
|
case MysqlTypes.NEWDECIMAL:
|
|
383
|
-
result = parseNewDecimal(parser, column);
|
|
399
|
+
result = parseNewDecimal(parser, column, zongji);
|
|
384
400
|
break;
|
|
385
401
|
case MysqlTypes.SET:
|
|
386
402
|
if (column.metadata.size === 8) {
|
|
@@ -454,12 +470,24 @@ export function readMysqlValue(
|
|
|
454
470
|
}
|
|
455
471
|
}
|
|
456
472
|
break;
|
|
457
|
-
case MysqlTypes.JSON:
|
|
473
|
+
case MysqlTypes.JSON: {
|
|
458
474
|
lengthSize = column.metadata['length_size'];
|
|
459
475
|
size = parser.parseUnsignedNumber(lengthSize);
|
|
460
476
|
buffer = parser.parseBuffer(size);
|
|
461
|
-
|
|
477
|
+
if (buffer.length === 0) {
|
|
478
|
+
result = null;
|
|
479
|
+
} else {
|
|
480
|
+
// Match mysql2 semantics: parsed JavaScript value by default,
|
|
481
|
+
// raw JSON text with the jsonStrings connection option. 64-bit
|
|
482
|
+
// integers stay exact: raw numerals in text mode, Numbers within
|
|
483
|
+
// the safe range (exact strings beyond it) in parsed mode.
|
|
484
|
+
const decoded = decodeJson(buffer);
|
|
485
|
+
const config = zongji && zongji.connection && zongji.connection.config;
|
|
486
|
+
result = config && config.jsonStrings ?
|
|
487
|
+
jsonToText(decoded) : convertBigInts(decoded);
|
|
488
|
+
}
|
|
462
489
|
break;
|
|
490
|
+
}
|
|
463
491
|
case MysqlTypes.GEOMETRY:
|
|
464
492
|
lengthSize = column.metadata['length_size'];
|
|
465
493
|
size = parser.parseUnsignedNumber(lengthSize);
|