@vlasky/zongji 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +30 -5
- package/example.js +5 -0
- package/index.d.ts +84 -2
- package/index.js +207 -65
- package/lib/binlog_event.js +49 -0
- package/lib/code_map.js +9 -1
- package/lib/common.js +71 -43
- package/lib/json_decode.js +61 -18
- package/lib/packet/binlog.js +7 -0
- package/lib/reader.js +4 -143
- package/lib/sequence/binlog.js +33 -8
- package/package.json +8 -5
- package/.travis.yml +0 -15
- package/docker-compose.yml +0 -33
- package/docker-test.sh +0 -11
- package/eslint.config.js +0 -35
- package/jsconfig.json +0 -15
- package/lib/packet/combinlog.js +0 -27
- package/lib/packet/index.js +0 -66
- package/scripts/start-mysql.sh +0 -28
package/lib/common.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import iconv from 'iconv-lite';
|
|
2
|
-
import decodeJson from './json_decode.js';
|
|
2
|
+
import decodeJson, { convertBigInts, jsonToText } from './json_decode.js';
|
|
3
3
|
import * as dtDecode from './datetime_decode.js';
|
|
4
|
-
import bigInt from 'big-integer';
|
|
5
4
|
|
|
6
5
|
export const MysqlTypes = {
|
|
7
6
|
DECIMAL: 0,
|
|
@@ -40,7 +39,27 @@ export const MysqlTypes = {
|
|
|
40
39
|
};
|
|
41
40
|
|
|
42
41
|
const TWO_TO_POWER_THIRTY_TWO = Math.pow(2, 32);
|
|
43
|
-
const TWO_TO_POWER_SIXTY_THREE =
|
|
42
|
+
const TWO_TO_POWER_SIXTY_THREE = 2n ** 63n;
|
|
43
|
+
const TWO_TO_POWER_SIXTY_FOUR = 2n ** 64n;
|
|
44
|
+
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
|
45
|
+
const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
|
|
46
|
+
|
|
47
|
+
// Combine two unsigned 32-bit halves into an exact unsigned 64-bit value.
|
|
48
|
+
// Returns a Number when at or below 2^53 - 1, otherwise an exact string.
|
|
49
|
+
export function unsignedInt64ToNumberOrString(high, low) {
|
|
50
|
+
const value = (BigInt(high) << 32n) | BigInt(low);
|
|
51
|
+
return value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Combine two unsigned 32-bit halves into an exact signed (two's
|
|
55
|
+
// complement) 64-bit value. Returns a Number when within the safe integer
|
|
56
|
+
// range, otherwise an exact string.
|
|
57
|
+
export function signedInt64ToNumberOrString(high, low) {
|
|
58
|
+
const value = BigInt.asIntN(64, (BigInt(high) << 32n) | BigInt(low));
|
|
59
|
+
return (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) ?
|
|
60
|
+
Number(value) : value.toString();
|
|
61
|
+
}
|
|
62
|
+
|
|
44
63
|
// This function will return a Number
|
|
45
64
|
// if the result < Math.MAX_SAFE_INTEGER or result > Math.MIN_SAFE_INTEGER,
|
|
46
65
|
// otherwise, will return a string.
|
|
@@ -48,11 +67,11 @@ export const parseUInt64 = function(parser) {
|
|
|
48
67
|
const low = parser.parseUnsignedNumber(4);
|
|
49
68
|
const high = parser.parseUnsignedNumber(4);
|
|
50
69
|
|
|
51
|
-
if (high >>> 21) { //
|
|
52
|
-
return
|
|
70
|
+
if (high >>> 21) { // value is 2^53 or more, exceeds safe Number range
|
|
71
|
+
return unsignedInt64ToNumberOrString(high, low);
|
|
53
72
|
}
|
|
54
73
|
|
|
55
|
-
return (high *
|
|
74
|
+
return (high * TWO_TO_POWER_THIRTY_TWO) + low;
|
|
56
75
|
};
|
|
57
76
|
|
|
58
77
|
export function parseUInt48(parser) {
|
|
@@ -115,12 +134,6 @@ export const parseIEEE754Float = function(high, low) {
|
|
|
115
134
|
}
|
|
116
135
|
};
|
|
117
136
|
|
|
118
|
-
export function getUInt32Value(input) {
|
|
119
|
-
// Last bit is not sign, it is part of value!
|
|
120
|
-
if (input & (1 << 31)) return Math.pow(2, 31) + (input & ((1 << 31) -1));
|
|
121
|
-
else return input;
|
|
122
|
-
};
|
|
123
|
-
|
|
124
137
|
const parseAnyInt = function(parser, column, columnSchema) {
|
|
125
138
|
let result, size;
|
|
126
139
|
switch (column.type) {
|
|
@@ -151,16 +164,13 @@ const parseAnyInt = function(parser, column, columnSchema) {
|
|
|
151
164
|
// Flip bits on negative signed integer
|
|
152
165
|
if (!int64 && (result & (1 << (length - 1)))) {
|
|
153
166
|
result = ((result ^ (Math.pow(2, length) - 1)) * -1) - 1;
|
|
154
|
-
} else if (int64
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
// Otherwise return a Number
|
|
162
|
-
else {
|
|
163
|
-
result = result.toJSNumber();
|
|
167
|
+
} else if (int64) {
|
|
168
|
+
const unsigned = BigInt(result);
|
|
169
|
+
if (unsigned >= TWO_TO_POWER_SIXTY_THREE) {
|
|
170
|
+
const signed = unsigned - TWO_TO_POWER_SIXTY_FOUR;
|
|
171
|
+
// Javascript Numbers only cover +-(2^53 - 1); return an exact
|
|
172
|
+
// string outside that range
|
|
173
|
+
result = signed >= MIN_SAFE_BIGINT ? Number(signed) : signed.toString();
|
|
164
174
|
}
|
|
165
175
|
}
|
|
166
176
|
}
|
|
@@ -185,7 +195,11 @@ const readIntBE = function(buf, offset, length, noAssert) {
|
|
|
185
195
|
// https://github.com/jeremycole/mysql_binlog/blob/master/lib/mysql_binlog/binlog_field_parser.rb
|
|
186
196
|
// Some more information about DECIMAL types:
|
|
187
197
|
// http://dev.mysql.com/doc/refman/5.5/en/precision-math-decimal-characteristics.html
|
|
188
|
-
|
|
198
|
+
// Returns the exact decimal value as a string (matching mysql2's default
|
|
199
|
+
// treatment of DECIMAL columns), or as a Number if the connection was
|
|
200
|
+
// configured with the mysql2 `decimalNumbers` option (which may lose
|
|
201
|
+
// precision beyond 15 significant digits).
|
|
202
|
+
const parseNewDecimal = function(parser, column, zongji) {
|
|
189
203
|
// Constants of format
|
|
190
204
|
const digitsPerInteger = 9;
|
|
191
205
|
const compressedBytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4];
|
|
@@ -197,38 +211,37 @@ const parseNewDecimal = function(parser, column) {
|
|
|
197
211
|
const compIntegral = integral - (uncompIntegral * digitsPerInteger);
|
|
198
212
|
const compFractional = scale - (uncompFractional * digitsPerInteger);
|
|
199
213
|
|
|
200
|
-
//
|
|
214
|
+
// Copy buffer portion: the sign bit is flipped in place below and the
|
|
215
|
+
// underlying packet buffer must not be mutated
|
|
201
216
|
const size = (uncompIntegral * 4) + compressedBytes[compIntegral] +
|
|
202
217
|
(uncompFractional * 4) + compressedBytes[compFractional];
|
|
203
|
-
const buffer =
|
|
218
|
+
const buffer = Buffer.from(
|
|
219
|
+
parser._buffer.subarray(parser._offset, parser._offset + size));
|
|
204
220
|
parser._offset += size; // Move binlog parser position forward
|
|
205
221
|
|
|
206
|
-
let str, mask;
|
|
207
222
|
let pos = 0;
|
|
208
223
|
const isPositive = (buffer.readUInt8(0) & (1 << 7)) === 128;
|
|
209
224
|
buffer.writeUInt8(buffer.readUInt8(0) ^ (1 << 7), 0);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
str = '';
|
|
213
|
-
mask = 0;
|
|
214
|
-
} else {
|
|
215
|
-
// Negative number
|
|
216
|
-
str = '-';
|
|
217
|
-
mask = -1;
|
|
218
|
-
}
|
|
225
|
+
// Negative numbers are stored with all bits flipped
|
|
226
|
+
const mask = isPositive ? 0 : -1;
|
|
219
227
|
|
|
220
228
|
// Build integer digits
|
|
229
|
+
let intDigits = '';
|
|
221
230
|
const compIntegralSize = compressedBytes[compIntegral];
|
|
222
231
|
if (compIntegralSize > 0) {
|
|
223
|
-
|
|
232
|
+
intDigits += (readIntBE(buffer, 0, compIntegralSize) ^ mask).toString(10);
|
|
224
233
|
pos += compIntegralSize;
|
|
225
234
|
}
|
|
226
235
|
|
|
227
236
|
for (let i = 0; i < uncompIntegral; i++) {
|
|
228
|
-
|
|
237
|
+
intDigits += zeroPad((buffer.readInt32BE(pos) ^ mask).toString(10), 9);
|
|
229
238
|
pos += 4;
|
|
230
239
|
}
|
|
231
240
|
|
|
241
|
+
// Strip leading zeros, keeping at least one digit
|
|
242
|
+
intDigits = intDigits.replace(/^0+(?=\d)/, '');
|
|
243
|
+
if (intDigits === '') intDigits = '0';
|
|
244
|
+
|
|
232
245
|
// Build fractional digits
|
|
233
246
|
let fractionDigits = '';
|
|
234
247
|
|
|
@@ -242,10 +255,13 @@ const parseNewDecimal = function(parser, column) {
|
|
|
242
255
|
fractionDigits += zeroPad((readIntBE(buffer, pos, compFractionalSize) ^ mask).toString(10), compFractional);
|
|
243
256
|
}
|
|
244
257
|
|
|
245
|
-
|
|
246
|
-
|
|
258
|
+
let str = (isPositive ? '' : '-') + intDigits;
|
|
259
|
+
if (scale > 0) {
|
|
260
|
+
str += '.' + fractionDigits;
|
|
261
|
+
}
|
|
247
262
|
|
|
248
|
-
|
|
263
|
+
const config = zongji && zongji.connection && zongji.connection.config;
|
|
264
|
+
return config && config.decimalNumbers ? parseFloat(str) : str;
|
|
249
265
|
};
|
|
250
266
|
|
|
251
267
|
// Did not work in place. Function cribbed from lines 311-363 of
|
|
@@ -380,7 +396,7 @@ export function readMysqlValue(
|
|
|
380
396
|
result = parseIEEE754Float(high, low);
|
|
381
397
|
break;
|
|
382
398
|
case MysqlTypes.NEWDECIMAL:
|
|
383
|
-
result = parseNewDecimal(parser, column);
|
|
399
|
+
result = parseNewDecimal(parser, column, zongji);
|
|
384
400
|
break;
|
|
385
401
|
case MysqlTypes.SET:
|
|
386
402
|
if (column.metadata.size === 8) {
|
|
@@ -454,12 +470,24 @@ export function readMysqlValue(
|
|
|
454
470
|
}
|
|
455
471
|
}
|
|
456
472
|
break;
|
|
457
|
-
case MysqlTypes.JSON:
|
|
473
|
+
case MysqlTypes.JSON: {
|
|
458
474
|
lengthSize = column.metadata['length_size'];
|
|
459
475
|
size = parser.parseUnsignedNumber(lengthSize);
|
|
460
476
|
buffer = parser.parseBuffer(size);
|
|
461
|
-
|
|
477
|
+
if (buffer.length === 0) {
|
|
478
|
+
result = null;
|
|
479
|
+
} else {
|
|
480
|
+
// Match mysql2 semantics: parsed JavaScript value by default,
|
|
481
|
+
// raw JSON text with the jsonStrings connection option. 64-bit
|
|
482
|
+
// integers stay exact: raw numerals in text mode, Numbers within
|
|
483
|
+
// the safe range (exact strings beyond it) in parsed mode.
|
|
484
|
+
const decoded = decodeJson(buffer);
|
|
485
|
+
const config = zongji && zongji.connection && zongji.connection.config;
|
|
486
|
+
result = config && config.jsonStrings ?
|
|
487
|
+
jsonToText(decoded) : convertBigInts(decoded);
|
|
488
|
+
}
|
|
462
489
|
break;
|
|
490
|
+
}
|
|
463
491
|
case MysqlTypes.GEOMETRY:
|
|
464
492
|
lengthSize = column.metadata['length_size'];
|
|
465
493
|
size = parser.parseUnsignedNumber(lengthSize);
|
package/lib/json_decode.js
CHANGED
|
@@ -21,11 +21,62 @@ const JSONB_LITERALS = [ null, true, false ];
|
|
|
21
21
|
// node-mysql prefixes binary string values
|
|
22
22
|
const VAR_STRING_PREFIX = 'base64:type253:';
|
|
23
23
|
|
|
24
|
+
// Decodes a JSONB binary buffer into JavaScript values (object, array,
|
|
25
|
+
// string, number, boolean, BigInt for 64-bit integers, or null). The
|
|
26
|
+
// caller converts BigInts and decides whether to serialise back to a JSON
|
|
27
|
+
// string (mysql2 `jsonStrings` option) via the helpers below.
|
|
24
28
|
export default function decodeJson(input) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
return parseBinaryBuffer(input);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
|
33
|
+
const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
|
|
34
|
+
|
|
35
|
+
// For parsed-value output: BigInts become Numbers when within the safe
|
|
36
|
+
// integer range, exact strings otherwise. Mutates objects/arrays in place
|
|
37
|
+
// (they were freshly created by parseBinaryBuffer).
|
|
38
|
+
export function convertBigInts(value) {
|
|
39
|
+
if (typeof value === 'bigint') {
|
|
40
|
+
return (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) ?
|
|
41
|
+
Number(value) : value.toString();
|
|
42
|
+
}
|
|
43
|
+
if (Array.isArray(value)) {
|
|
44
|
+
for (let i = 0; i < value.length; i++) {
|
|
45
|
+
value[i] = convertBigInts(value[i]);
|
|
46
|
+
}
|
|
47
|
+
return value;
|
|
48
|
+
}
|
|
49
|
+
if (value !== null && typeof value === 'object') {
|
|
50
|
+
for (const key of Object.keys(value)) {
|
|
51
|
+
value[key] = convertBigInts(value[key]);
|
|
52
|
+
}
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
return value;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// For jsonStrings output: serialise to JSON text with 64-bit integers as
|
|
59
|
+
// raw numerals (JSON.stringify cannot represent BigInt), so the text
|
|
60
|
+
// matches what the server would print for the column.
|
|
61
|
+
export function jsonToText(value) {
|
|
62
|
+
if (value === null) {
|
|
63
|
+
return 'null';
|
|
64
|
+
}
|
|
65
|
+
switch (typeof value) {
|
|
66
|
+
case 'bigint':
|
|
67
|
+
case 'number':
|
|
68
|
+
case 'boolean':
|
|
69
|
+
return String(value);
|
|
70
|
+
case 'string':
|
|
71
|
+
return JSON.stringify(value);
|
|
72
|
+
default:
|
|
73
|
+
if (Array.isArray(value)) {
|
|
74
|
+
return '[' + value.map(jsonToText).join(', ') + ']';
|
|
75
|
+
}
|
|
76
|
+
return '{' + Object.keys(value).map(key =>
|
|
77
|
+
JSON.stringify(key) + ': ' + jsonToText(value[key])
|
|
78
|
+
).join(', ') + '}';
|
|
79
|
+
}
|
|
29
80
|
}
|
|
30
81
|
|
|
31
82
|
/*
|
|
@@ -112,25 +163,17 @@ function parseBinaryBuffer(input, offset, parentValueOffset, readUInt, isLarge)
|
|
|
112
163
|
result = isLarge ? input.readUInt32LE(offset + 1) : input.readUInt32LE(valueOffset + 1);
|
|
113
164
|
break;
|
|
114
165
|
case JSONB_TYPE_INT64: {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const mask = Math.pow(2, 32) - 1;
|
|
121
|
-
high = common.sliceBits(high ^ mask, 0, 21);
|
|
122
|
-
low = low ^ mask;
|
|
123
|
-
result =
|
|
124
|
-
((high * Math.pow(2, 32)) * - 1) - common.getUInt32Value(low) - 1;
|
|
125
|
-
} else {
|
|
126
|
-
result = (high * Math.pow(2,32)) + low;
|
|
127
|
-
}
|
|
166
|
+
const low = input.readUInt32LE(valueOffset + 1);
|
|
167
|
+
const high = input.readUInt32LE(valueOffset + 5);
|
|
168
|
+
// Exact; converted to Number / string / raw JSON numeral by the
|
|
169
|
+
// consumers below
|
|
170
|
+
result = BigInt.asIntN(64, (BigInt(high) << 32n) | BigInt(low));
|
|
128
171
|
break;
|
|
129
172
|
}
|
|
130
173
|
case JSONB_TYPE_UINT64: {
|
|
131
174
|
const low = input.readUInt32LE(valueOffset + 1);
|
|
132
175
|
const high = input.readUInt32LE(valueOffset + 5);
|
|
133
|
-
result = (high
|
|
176
|
+
result = (BigInt(high) << 32n) | BigInt(low);
|
|
134
177
|
break;
|
|
135
178
|
}
|
|
136
179
|
case JSONB_TYPE_OPAQUE: {
|
package/lib/packet/binlog.js
CHANGED
|
@@ -32,6 +32,13 @@ export default function initBinlogPacketClass(zongji) {
|
|
|
32
32
|
const EventClass = getEventClass(eventType);
|
|
33
33
|
this.eventName = EventClass.name;
|
|
34
34
|
|
|
35
|
+
// Warn (once per instance) about event types that carry row changes
|
|
36
|
+
// zongji cannot decode. This must happen before event filtering so
|
|
37
|
+
// the data loss is loud even when the type is not included.
|
|
38
|
+
if (EventClass.unsupportedReason) {
|
|
39
|
+
zongji._warnUnsupportedEvent(EventClass);
|
|
40
|
+
}
|
|
41
|
+
|
|
35
42
|
yield;
|
|
36
43
|
|
|
37
44
|
try {
|
package/lib/reader.js
CHANGED
|
@@ -1,145 +1,6 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
export const UNSIGNED_SHORT_COLUMN = 252;
|
|
5
|
-
export const UNSIGNED_INT24_COLUMN = 253;
|
|
6
|
-
export const UNSIGNED_INT64_COLUMN = 254;
|
|
7
|
-
|
|
8
|
-
const padWith = function(val, length) {
|
|
9
|
-
const bits = val.split('');
|
|
10
|
-
if (bits.length < length) {
|
|
11
|
-
const left = length - bits.length;
|
|
12
|
-
for (let j = left - 1; j >= 0; j--) {
|
|
13
|
-
bits.unshift('0');
|
|
14
|
-
}
|
|
15
|
-
val = bits.join('');
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
return val;
|
|
19
|
-
};
|
|
20
|
-
|
|
21
|
-
class BufferReader {
|
|
22
|
-
constructor(buffer) {
|
|
23
|
-
this.buffer = buffer;
|
|
24
|
-
this.position = 0;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
readUInt8() {
|
|
28
|
-
const pos = this.position;
|
|
29
|
-
this.position += 1;
|
|
30
|
-
|
|
31
|
-
return this.buffer.readUInt8(pos);
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
readUInt16() {
|
|
35
|
-
const pos = this.position;
|
|
36
|
-
this.position += 2;
|
|
37
|
-
|
|
38
|
-
return this.buffer.readUInt16LE(pos);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
readUInt32() {
|
|
42
|
-
const pos = this.position;
|
|
43
|
-
this.position += 4;
|
|
44
|
-
|
|
45
|
-
return this.buffer.readUInt32LE(pos);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
readUInt24() {
|
|
49
|
-
const low = this.readUInt16();
|
|
50
|
-
const high = this.readUInt8();
|
|
51
|
-
return (high << 16) + low;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
readUInt64() {
|
|
55
|
-
const pos = this.position;
|
|
56
|
-
this.position += 8;
|
|
57
|
-
|
|
58
|
-
// from http://stackoverflow.com/questions/17687307/convert-a-64bit-little-endian-integer-to-number
|
|
59
|
-
return this.buffer.readInt32LE(pos) +
|
|
60
|
-
0x100000000 * this.buffer.readUInt32LE(pos + 4);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
readString() {
|
|
64
|
-
const strBuf = this.buffer.slice(this.position);
|
|
65
|
-
this.position = this.buffer.length;
|
|
66
|
-
|
|
67
|
-
return strBuf.toString('ascii');
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
readStringInBytes(length) {
|
|
71
|
-
const strBuf = this.buffer.slice(this.position, this.position + length);
|
|
72
|
-
this.position += length;
|
|
73
|
-
|
|
74
|
-
return strBuf.toString('ascii');
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
readHexInBytes(length) {
|
|
78
|
-
const buf = this.buffer.slice(this.position, this.position + length);
|
|
79
|
-
this.position += length;
|
|
80
|
-
|
|
81
|
-
return buf.toString('hex');
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
readBytesArray(length) {
|
|
85
|
-
const result = [];
|
|
86
|
-
const hexString = this.readHexInBytes(length);
|
|
87
|
-
for (let i = 0; i < hexString.length; i = i + 2) {
|
|
88
|
-
result.push(parseInt(hexString.substr(i, 2), 16));
|
|
89
|
-
}
|
|
90
|
-
return result;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Read a variable-length "Length Coded Binary" integer. This is derived
|
|
94
|
-
// from the MySQL protocol, and re-used in the binary log format. This
|
|
95
|
-
// format uses the first byte to alternately store the actual value for
|
|
96
|
-
// integer values <= 250, or to encode the number of following bytes
|
|
97
|
-
// used to store the actual value, which can be 2, 3, or 8. It also
|
|
98
|
-
// includes support for SQL NULL as a special case.
|
|
99
|
-
readVariant() {
|
|
100
|
-
let result = null;
|
|
101
|
-
const firstByte = this.readUInt8();
|
|
102
|
-
|
|
103
|
-
if (firstByte < UNSIGNED_CHAR_COLUMN) {
|
|
104
|
-
result = firstByte;
|
|
105
|
-
} else if (firstByte === NULL_COLUMN) {
|
|
106
|
-
result = null;
|
|
107
|
-
} else if (firstByte === UNSIGNED_SHORT_COLUMN) {
|
|
108
|
-
result = this.readUInt16();
|
|
109
|
-
} else if (firstByte === UNSIGNED_INT24_COLUMN) {
|
|
110
|
-
result = this.readUInt24();
|
|
111
|
-
} else if (firstByte === UNSIGNED_INT64_COLUMN) {
|
|
112
|
-
result = this.readUInt64();
|
|
113
|
-
} else {
|
|
114
|
-
throw new Error('Invalid variable-length integer');
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
return result;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// Read an arbitrary-length bitmap, provided its length.
|
|
121
|
-
// Returns an array of true/false values.
|
|
122
|
-
readBitArray(length) {
|
|
123
|
-
const size = Math.floor((length + 7) / 8);
|
|
124
|
-
|
|
125
|
-
const bytes = [];
|
|
126
|
-
for (let i = size - 1; i >= 0; i--) {
|
|
127
|
-
bytes.unshift(this.readUInt8());
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
const bitmap = [];
|
|
131
|
-
const bitmapStr = bytes.map(function(aByte) {
|
|
132
|
-
return padWith(aByte.toString(2), 8);
|
|
133
|
-
}).join('');
|
|
134
|
-
|
|
135
|
-
for (let k = bitmapStr.length - 1; k >= 0; k--) {
|
|
136
|
-
bitmap.push(bitmapStr[k] === '1');
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
return bitmap.slice(0, length);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
1
|
+
// Minimal parser over a mysql2 packet (or a bare buffer via append()),
|
|
2
|
+
// providing the subset of the node-mysql Parser interface that the binlog
|
|
3
|
+
// event classes consume.
|
|
143
4
|
class Parser {
|
|
144
5
|
constructor(packet) {
|
|
145
6
|
this.packet = null;
|
|
@@ -244,4 +105,4 @@ class Parser {
|
|
|
244
105
|
}
|
|
245
106
|
}
|
|
246
107
|
|
|
247
|
-
export {
|
|
108
|
+
export { Parser };
|
package/lib/sequence/binlog.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventEmitter } from 'events';
|
|
2
|
-
import
|
|
2
|
+
import initBinlogPacketClass from '../packet/binlog.js';
|
|
3
3
|
import { Parser } from '../reader.js';
|
|
4
4
|
|
|
5
5
|
const BINLOG_DUMP_COMMAND = 0x12;
|
|
@@ -137,16 +137,41 @@ export default function initBinlogClass(zongji) {
|
|
|
137
137
|
const eventName = binlogPacket.eventName
|
|
138
138
|
? binlogPacket.eventName.toLowerCase()
|
|
139
139
|
: 'unknown';
|
|
140
|
-
if (zongji._skipEvent(eventName)) {
|
|
141
|
-
return Binlog.prototype.binlogData;
|
|
142
|
-
}
|
|
143
140
|
|
|
144
141
|
let event;
|
|
145
142
|
let error;
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
143
|
+
|
|
144
|
+
// Track the current transaction GTID before event filtering, so
|
|
145
|
+
// subsequent events can carry event.gtid even when 'gtid' events
|
|
146
|
+
// themselves are excluded by includeEvents
|
|
147
|
+
const isGtidEvent =
|
|
148
|
+
eventName === 'gtid' || eventName === 'anonymousgtid';
|
|
149
|
+
if (isGtidEvent) {
|
|
150
|
+
try {
|
|
151
|
+
event = binlogPacket.getEvent();
|
|
152
|
+
// Anonymous transactions have no usable GTID; do not let a
|
|
153
|
+
// previous transaction's GTID leak onto their events
|
|
154
|
+
zongji._currentGtid =
|
|
155
|
+
eventName === 'gtid' ? event.gtid : undefined;
|
|
156
|
+
} catch (err) {
|
|
157
|
+
zongji._currentGtid = undefined;
|
|
158
|
+
error = err;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// A GTID parse failure corrupts transaction attribution for the
|
|
163
|
+
// whole following transaction, so it must surface even when the
|
|
164
|
+
// event type itself is filtered out
|
|
165
|
+
if (zongji._skipEvent(eventName) && !error) {
|
|
166
|
+
return Binlog.prototype.binlogData;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (!isGtidEvent && !error) {
|
|
170
|
+
try {
|
|
171
|
+
event = binlogPacket.getEvent();
|
|
172
|
+
} catch (err) {
|
|
173
|
+
error = err;
|
|
174
|
+
}
|
|
150
175
|
}
|
|
151
176
|
this._callback.call(this, error, event);
|
|
152
177
|
}
|
package/package.json
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vlasky/zongji",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"description": "MySQL binlog-based change data capture (CDC) for Node.js",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"types": "index.d.ts",
|
|
8
|
-
"
|
|
9
|
-
"
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./index.d.ts",
|
|
11
|
+
"default": "./index.js"
|
|
12
|
+
}
|
|
10
13
|
},
|
|
11
14
|
"scripts": {
|
|
12
15
|
"pretest": "bash scripts/start-mysql.sh",
|
|
13
16
|
"test": "tap --bail --disable-coverage --jobs=1 test/*.js",
|
|
17
|
+
"test:only": "tap --bail --disable-coverage --jobs=1 test/*.js",
|
|
14
18
|
"lint": "eslint .",
|
|
15
19
|
"checkjs": "tsc --project jsconfig.json"
|
|
16
20
|
},
|
|
@@ -42,8 +46,7 @@
|
|
|
42
46
|
"tap": "^21.5.0"
|
|
43
47
|
},
|
|
44
48
|
"dependencies": {
|
|
45
|
-
"big-integer": "1.6.52",
|
|
46
49
|
"iconv-lite": "0.7.2",
|
|
47
|
-
"mysql2": "^3.
|
|
50
|
+
"mysql2": "^3.22.5"
|
|
48
51
|
}
|
|
49
52
|
}
|
package/.travis.yml
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
language: node_js
|
|
2
|
-
sudo: required
|
|
3
|
-
dist: trusty
|
|
4
|
-
node_js:
|
|
5
|
-
- "8"
|
|
6
|
-
- "12"
|
|
7
|
-
services:
|
|
8
|
-
- docker
|
|
9
|
-
before_script:
|
|
10
|
-
- npm run lint
|
|
11
|
-
- docker run -p 3355:3306 --name mysql-55 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.5 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
|
|
12
|
-
- docker run -p 3356:3306 --name mysql-56 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.6 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
|
|
13
|
-
- docker run -p 3357:3306 --name mysql-57 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.7 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
|
|
14
|
-
script:
|
|
15
|
-
- ./test/travis/runner.sh
|
package/docker-compose.yml
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
services:
|
|
3
|
-
mysql57:
|
|
4
|
-
image: mysql:5.7
|
|
5
|
-
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
6
|
-
networks:
|
|
7
|
-
default:
|
|
8
|
-
aliases:
|
|
9
|
-
- mysql57
|
|
10
|
-
environment:
|
|
11
|
-
MYSQL_ROOT_PASSWORD: secret
|
|
12
|
-
|
|
13
|
-
mysql80:
|
|
14
|
-
image: mysql:8.0
|
|
15
|
-
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
16
|
-
networks:
|
|
17
|
-
default:
|
|
18
|
-
aliases:
|
|
19
|
-
- mysql80
|
|
20
|
-
environment:
|
|
21
|
-
MYSQL_ROOT_PASSWORD: secret
|
|
22
|
-
|
|
23
|
-
mysql84:
|
|
24
|
-
image: mysql:8.4
|
|
25
|
-
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
26
|
-
ports:
|
|
27
|
-
- "3306:3306"
|
|
28
|
-
networks:
|
|
29
|
-
default:
|
|
30
|
-
aliases:
|
|
31
|
-
- mysql84
|
|
32
|
-
environment:
|
|
33
|
-
MYSQL_ROOT_PASSWORD: secret
|
package/docker-test.sh
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
MYSQL_HOSTS="mysql57 mysql80 mysql84"
|
|
3
|
-
|
|
4
|
-
for hostname in ${MYSQL_HOSTS}; do
|
|
5
|
-
echo $hostname + node 18
|
|
6
|
-
docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:18 npm test
|
|
7
|
-
echo $hostname + node 20
|
|
8
|
-
docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:20 npm test
|
|
9
|
-
echo $hostname + node 22
|
|
10
|
-
docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:22 npm test
|
|
11
|
-
done
|
package/eslint.config.js
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import js from '@eslint/js';
|
|
2
|
-
|
|
3
|
-
export default [
|
|
4
|
-
{ ignores: ['.tap/'] },
|
|
5
|
-
js.configs.recommended,
|
|
6
|
-
{
|
|
7
|
-
languageOptions: {
|
|
8
|
-
ecmaVersion: 2022,
|
|
9
|
-
sourceType: 'module',
|
|
10
|
-
globals: {
|
|
11
|
-
Buffer: 'readonly',
|
|
12
|
-
clearImmediate: 'readonly',
|
|
13
|
-
clearInterval: 'readonly',
|
|
14
|
-
clearTimeout: 'readonly',
|
|
15
|
-
console: 'readonly',
|
|
16
|
-
process: 'readonly',
|
|
17
|
-
setImmediate: 'readonly',
|
|
18
|
-
setInterval: 'readonly',
|
|
19
|
-
setTimeout: 'readonly',
|
|
20
|
-
},
|
|
21
|
-
},
|
|
22
|
-
rules: {
|
|
23
|
-
'comma-dangle': ['warn', 'only-multiline'],
|
|
24
|
-
'eol-last': ['error'],
|
|
25
|
-
'keyword-spacing': ['error', { before: true }],
|
|
26
|
-
'no-console': 'off',
|
|
27
|
-
'no-trailing-spaces': ['error', { skipBlankLines: true }],
|
|
28
|
-
'no-unused-vars': 'warn',
|
|
29
|
-
'no-var': 'warn',
|
|
30
|
-
'quotes': ['warn', 'single', 'avoid-escape'],
|
|
31
|
-
'semi': ['error', 'always'],
|
|
32
|
-
'space-before-blocks': 'error',
|
|
33
|
-
},
|
|
34
|
-
},
|
|
35
|
-
];
|
package/jsconfig.json
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"checkJs": true,
|
|
4
|
-
"allowJs": true,
|
|
5
|
-
"noEmit": true,
|
|
6
|
-
"target": "ES2022",
|
|
7
|
-
"module": "NodeNext",
|
|
8
|
-
"moduleResolution": "NodeNext",
|
|
9
|
-
"skipDefaultLibCheck": true,
|
|
10
|
-
"maxNodeModuleJsDepth": 0,
|
|
11
|
-
"types": ["node"]
|
|
12
|
-
},
|
|
13
|
-
"include": ["index.js", "index.d.ts", "lib/**/*.js"],
|
|
14
|
-
"exclude": ["node_modules", ".tap"]
|
|
15
|
-
}
|