@vlasky/zongji 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +35 -5
  3. package/example.js +5 -0
  4. package/index.d.ts +84 -2
  5. package/index.js +167 -59
  6. package/lib/binlog_event.js +39 -0
  7. package/lib/code_map.js +9 -1
  8. package/lib/common.js +71 -43
  9. package/lib/json_decode.js +61 -18
  10. package/lib/packet/binlog.js +7 -0
  11. package/lib/reader.js +4 -143
  12. package/lib/sequence/binlog.js +29 -7
  13. package/package.json +9 -6
  14. package/.claude/settings.local.json +0 -16
  15. package/.mcp.json +0 -17
  16. package/.tap/processinfo/84de694d-068b-4ac7-b4d8-f835c2122b2d.json +0 -892
  17. package/.tap/processinfo/a1e9627e-66eb-412c-aaa5-079a2eb9f385.json +0 -891
  18. package/.tap/processinfo/abc34c6f-f4aa-4405-8f9a-947048be1df2.json +0 -892
  19. package/.tap/processinfo/affe6c5b-a34d-41b9-bb73-4b14d35138de.json +0 -891
  20. package/.tap/processinfo/bf82449d-cdeb-478b-9506-b939a9506bf2.json +0 -883
  21. package/.tap/processinfo/dd98c508-7067-4342-bdf5-63664b630701.json +0 -892
  22. package/.tap/test-results/test_await_end.js.tap +0 -19
  23. package/.tap/test-results/test_await_end2.js.tap +0 -27
  24. package/.tap/test-results/test_debug_init.js.tap +0 -23
  25. package/.tap/test-results/test_debug_init2.js.tap +0 -35
  26. package/.tap/test-results/test_invalid_host.js.tap +0 -34
  27. package/.tap/test-results/test_minimal.js.tap +0 -26
  28. package/.tap/test-results/test_queryseq.js.tap +0 -14
  29. package/.tap/test-results/test_timing.js.tap +0 -17
  30. package/.tap/test-results/test_unref.js.tap +0 -15
  31. package/.travis.yml +0 -15
  32. package/CLAUDE.md +0 -99
  33. package/GTID.md +0 -54
  34. package/TODO.md +0 -43
  35. package/docker-compose.yml +0 -33
  36. package/docker-test.sh +0 -11
  37. package/eslint.config.js +0 -35
  38. package/jsconfig.json +0 -15
  39. package/lib/packet/combinlog.js +0 -27
  40. package/lib/packet/index.js +0 -66
  41. package/notes.md +0 -77
  42. package/scripts/start-mysql.sh +0 -28
  43. package/workerthreads.md +0 -173
package/index.js CHANGED
@@ -7,23 +7,18 @@ const ConnectionConfigMap = {
7
7
  'Pool': obj => obj.config.connectionConfig,
8
8
  };
9
9
 
10
- const TableInfoQueryTemplate = `SELECT
10
+ const TableInfoQuery = `SELECT
11
11
  COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME,
12
12
  COLUMN_COMMENT, COLUMN_TYPE
13
- FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'
13
+ FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=?
14
14
  ORDER BY ORDINAL_POSITION`;
15
15
 
16
- // Simple format function to replace %s placeholders
17
- function formatSql(template, ...args) {
18
- let i = 0;
19
- return template.replace(/%s/g, () => args[i++]);
20
- }
21
-
22
16
  class ZongJi extends EventEmitter {
23
17
  constructor(dsn) {
24
18
  super();
25
19
 
26
20
  this._pendingErrors = [];
21
+ this._pendingErrorTimer = null;
27
22
  this.on('newListener', (event) => {
28
23
  if (event !== 'error' || this._pendingErrors.length === 0) {
29
24
  return;
@@ -39,8 +34,12 @@ class ZongJi extends EventEmitter {
39
34
  this._filters({});
40
35
  this.ctrlCallbacks = [];
41
36
  this.tableMap = {};
37
+ this._warnedUnsupported = new Set();
38
+ this._currentGtid = undefined;
42
39
  this.ready = false;
43
40
  this.stopped = false;
41
+ this._starting = false;
42
+ this._startEpoch = 0;
44
43
  this.useChecksum = false;
45
44
 
46
45
  this._dsn = dsn;
@@ -54,7 +53,12 @@ class ZongJi extends EventEmitter {
54
53
  const createConnection = (options) => {
55
54
  const emitError = (err) => {
56
55
  if (this.listenerCount('error') === 0) {
56
+ // Buffer errors that fire before the caller attaches a listener
57
+ // (typically within the same tick as construction/start). If no
58
+ // listener ever appears, fall back to EventEmitter's default
59
+ // behaviour (throw) rather than swallowing failures silently.
57
60
  this._pendingErrors.push(err);
61
+ this._schedulePendingErrorCheck();
58
62
  return;
59
63
  }
60
64
  this.emit('error', err);
@@ -95,6 +99,23 @@ class ZongJi extends EventEmitter {
95
99
  this.connection = createConnection(binlogDsn);
96
100
  }
97
101
 
102
+ _schedulePendingErrorCheck() {
103
+ if (this._pendingErrorTimer) {
104
+ return;
105
+ }
106
+ this._pendingErrorTimer = setImmediate(() => {
107
+ this._pendingErrorTimer = null;
108
+ if (this.listenerCount('error') > 0 || this._pendingErrors.length === 0) {
109
+ return;
110
+ }
111
+ const pending = this._pendingErrors.slice();
112
+ this._pendingErrors.length = 0;
113
+ // No 'error' listener was attached within a macrotask; re-emit so
114
+ // Node's unhandled 'error' semantics apply (throws)
115
+ pending.forEach(err => this.emit('error', err));
116
+ });
117
+ }
118
+
98
119
  _isChecksumEnabled(next) {
99
120
  const SelectChecksumParamSql = 'select @@GLOBAL.binlog_checksum as checksum';
100
121
  const SetChecksumSql = 'set @master_binlog_checksum=@@global.binlog_checksum';
@@ -184,19 +205,28 @@ class ZongJi extends EventEmitter {
184
205
  }
185
206
 
186
207
  _fetchTableInfo(tableMapEvent, next) {
187
- const sql = formatSql(TableInfoQueryTemplate,
188
- tableMapEvent.schemaName, tableMapEvent.tableName);
189
-
190
208
  if (!this.ctrlConnection ||
191
209
  this.ctrlConnection.state === 'disconnected' ||
192
210
  this.ctrlConnection._fatalError ||
193
211
  this.ctrlConnection._protocolError ||
194
212
  this.ctrlConnection._closing) {
213
+ // The binlog connection stays paused, so processing has halted.
214
+ // During stop() that is expected; otherwise it must not be silent.
215
+ if (!this.stopped) {
216
+ this.emit('error', new Error(
217
+ 'Control connection unavailable while fetching column metadata ' +
218
+ 'for ' + tableMapEvent.schemaName + '.' + tableMapEvent.tableName +
219
+ '. Binlog processing has halted; call stop() then start() to resume.'));
220
+ }
195
221
  return;
196
222
  }
197
223
 
224
+ const params = [tableMapEvent.schemaName, tableMapEvent.tableName];
198
225
  try {
199
- this.ctrlConnection.query(sql, (err, rows) => {
226
+ // execute() uses a server-side prepared statement: parameters are sent
227
+ // out-of-band (never spliced into SQL text) and the statement is cached
228
+ // per connection, so repeated metadata lookups avoid re-parsing.
229
+ this.ctrlConnection.execute(TableInfoQuery, params, (err, rows) => {
200
230
  if (err) {
201
231
  // Errors should be emitted
202
232
  this.emit('error', err);
@@ -234,12 +264,14 @@ class ZongJi extends EventEmitter {
234
264
  filename,
235
265
  position,
236
266
  startAtEnd,
267
+ nonBlock,
237
268
  } = {}) {
238
269
  this.options = {
239
270
  serverId,
240
271
  filename,
241
272
  position,
242
273
  startAtEnd,
274
+ nonBlock,
243
275
  };
244
276
  }
245
277
 
@@ -257,6 +289,28 @@ class ZongJi extends EventEmitter {
257
289
  includeSchema,
258
290
  excludeSchema,
259
291
  };
292
+
293
+ // Precompiled lookups so per-event filtering is O(1). Only own keys of
294
+ // the schema objects are considered (no prototype leakage).
295
+ const compileSchemaFilter = (schema) => {
296
+ if (schema === undefined || schema === null) {
297
+ return undefined;
298
+ }
299
+ const compiled = new Map();
300
+ for (const database of Object.keys(schema)) {
301
+ const tables = schema[database];
302
+ compiled.set(database, tables === true ?
303
+ true : new Set(Array.isArray(tables) ? tables : []));
304
+ }
305
+ return compiled;
306
+ };
307
+
308
+ this._includeEventsSet = includeEvents === undefined ?
309
+ undefined : new Set(Array.isArray(includeEvents) ? includeEvents : []);
310
+ this._excludeEventsSet =
311
+ new Set(Array.isArray(excludeEvents) ? excludeEvents : []);
312
+ this._includeSchemaMap = compileSchemaFilter(includeSchema);
313
+ this._excludeSchemaMap = compileSchemaFilter(excludeSchema) || new Map();
260
314
  }
261
315
 
262
316
  get(name) {
@@ -289,6 +343,22 @@ class ZongJi extends EventEmitter {
289
343
  return;
290
344
  }
291
345
 
346
+ // A duplicate start() while one is already initialising is ignored;
347
+ // the first call completes. But if stop() intervened, this start()
348
+ // must proceed as a restart: the epoch below makes the stale
349
+ // initialisation chain abort, so exactly one binlog dump command is
350
+ // ever enqueued.
351
+ if (this._starting && !this.stopped) {
352
+ return;
353
+ }
354
+ this._starting = true;
355
+ this._startEpoch += 1;
356
+ const epoch = this._startEpoch;
357
+
358
+ // A resumed stream must not attribute early events to a GTID seen
359
+ // before the restart
360
+ this._currentGtid = undefined;
361
+
292
362
  this.stopped = false;
293
363
 
294
364
  if (!this.connection || !this.ctrlConnection) {
@@ -330,6 +400,16 @@ class ZongJi extends EventEmitter {
330
400
  });
331
401
  };
332
402
 
403
+ // Attach the current transaction's GTID (tracked at the packet layer,
404
+ // even when 'gtid' events are filtered out). Gtid/AnonymousGtid events
405
+ // keep their own parsed value.
406
+ const attachGtid = (event) => {
407
+ if (!('gtid' in event)) {
408
+ event.gtid = this._currentGtid;
409
+ }
410
+ return event;
411
+ };
412
+
333
413
  const binlogHandler = (error, event) => {
334
414
  if (error) {
335
415
  return this.emit('error', error);
@@ -347,6 +427,7 @@ class ZongJi extends EventEmitter {
347
427
  if (!tableMap || tableMap.tableName !== event.tableName || tableMap.columns.length !== event.columnCount) {
348
428
  if (!this.connection) return;
349
429
  this.connection.pause();
430
+ attachGtid(event);
350
431
  this._fetchTableInfo(event, () => {
351
432
  // merge the column info with metadata
352
433
  event.updateColumnInfo();
@@ -364,7 +445,7 @@ class ZongJi extends EventEmitter {
364
445
  break;
365
446
  }
366
447
  this.options.position = event.nextPosition;
367
- this.emit('binlog', event);
448
+ this.emit('binlog', attachGtid(event));
368
449
  };
369
450
 
370
451
  let promises = [new Promise(testChecksum)];
@@ -375,7 +456,13 @@ class ZongJi extends EventEmitter {
375
456
 
376
457
  Promise.all(promises)
377
458
  .then(() => {
378
- // Check if stop() was called while promises were pending
459
+ // Abort if a newer start() superseded this one (after a stop())
460
+ // while promises were pending
461
+ if (epoch !== this._startEpoch) {
462
+ return;
463
+ }
464
+ this._starting = false;
465
+ // Abort if stop() was called while promises were pending
379
466
  if (this.stopped) {
380
467
  return;
381
468
  }
@@ -384,8 +471,9 @@ class ZongJi extends EventEmitter {
384
471
  this.ready = true;
385
472
  this.emit('ready');
386
473
 
387
- // Final check right before addCommand - connection may have been destroyed
388
- if (this.stopped) {
474
+ // Final check right before addCommand - connection may have been
475
+ // destroyed or this start() superseded by a ready handler
476
+ if (this.stopped || epoch !== this._startEpoch) {
389
477
  return;
390
478
  }
391
479
 
@@ -413,6 +501,10 @@ class ZongJi extends EventEmitter {
413
501
  this.connection.addCommand(new this.BinlogClass(binlogHandler));
414
502
  })
415
503
  .catch(err => {
504
+ if (epoch !== this._startEpoch) {
505
+ return;
506
+ }
507
+ this._starting = false;
416
508
  this.emit('error', err);
417
509
  });
418
510
 
@@ -427,10 +519,25 @@ class ZongJi extends EventEmitter {
427
519
  return;
428
520
  }
429
521
 
522
+ // Errors emitted by a connection we are deliberately destroying are
523
+ // teardown noise (e.g. in-flight queries failing with
524
+ // ERR_STREAM_WRITE_AFTER_END); do not forward them to the caller
525
+ const silenceErrors = (conn) => {
526
+ conn.removeAllListeners('error');
527
+ conn.on('error', () => {});
528
+ };
529
+
530
+ // Capture the connections this stop() owns: finish() may fire from an
531
+ // async callback after a subsequent start() has already created
532
+ // replacement connections, and must not touch those
533
+ const ctrlConnection = this.ctrlConnection;
534
+ const ctrlToClose = this.ctrlConnectionOwner ? ctrlConnection : null;
535
+
430
536
  // Binary log connection does not end with destroy()
431
537
  let connectionThreadId = null;
432
538
  if (this.connection) {
433
539
  connectionThreadId = this.connection.threadId;
540
+ silenceErrors(this.connection);
434
541
  this.connection.destroy();
435
542
  // @ts-ignore - internal mysql2 API
436
543
  if (this.connection.stream && typeof this.connection.stream.unref === 'function') {
@@ -443,24 +550,29 @@ class ZongJi extends EventEmitter {
443
550
  const finish = () => {
444
551
  if (finished) return;
445
552
  finished = true;
446
- if (this.ctrlConnectionOwner && this.ctrlConnection) {
447
- this.ctrlConnection.destroy();
553
+ if (ctrlToClose) {
554
+ silenceErrors(ctrlToClose);
555
+ ctrlToClose.destroy();
448
556
  // @ts-ignore - internal mysql2 API
449
- if (this.ctrlConnection.stream && typeof this.ctrlConnection.stream.unref === 'function') {
557
+ if (ctrlToClose.stream && typeof ctrlToClose.stream.unref === 'function') {
450
558
  // @ts-ignore - internal mysql2 API
451
- this.ctrlConnection.stream.unref();
559
+ ctrlToClose.stream.unref();
560
+ }
561
+ if (this.ctrlConnection === ctrlToClose) {
562
+ this.ctrlConnection = null;
452
563
  }
453
- this.ctrlConnection = null;
454
564
  }
455
565
  this.emit('stopped');
456
566
  };
457
567
 
458
- if (!this.ctrlConnection ||
459
- this.ctrlConnection.state === 'disconnected' ||
460
- this.ctrlConnection._fatalError ||
461
- this.ctrlConnection._protocolError ||
462
- this.ctrlConnection._closing) {
463
- this.ctrlConnection = null;
568
+ if (!ctrlConnection ||
569
+ ctrlConnection.state === 'disconnected' ||
570
+ ctrlConnection._fatalError ||
571
+ ctrlConnection._protocolError ||
572
+ ctrlConnection._closing) {
573
+ if (this.ctrlConnection === ctrlConnection) {
574
+ this.ctrlConnection = null;
575
+ }
464
576
  return finish();
465
577
  }
466
578
 
@@ -470,7 +582,7 @@ class ZongJi extends EventEmitter {
470
582
 
471
583
  const killTimeout = setTimeout(finish, 1000);
472
584
  try {
473
- this.ctrlConnection.query(
585
+ ctrlConnection.query(
474
586
  'KILL ' + connectionThreadId,
475
587
  () => {
476
588
  clearTimeout(killTimeout);
@@ -483,44 +595,40 @@ class ZongJi extends EventEmitter {
483
595
  }
484
596
  }
485
597
 
598
+ // Emit an error the first time an undecodable event type arrives so that
599
+ // dropped row changes (e.g. compressed transactions) are never silent.
600
+ _warnUnsupportedEvent(EventClass) {
601
+ if (this._warnedUnsupported.has(EventClass.name)) {
602
+ return;
603
+ }
604
+ this._warnedUnsupported.add(EventClass.name);
605
+ this.emit('error', new Error(EventClass.unsupportedReason));
606
+ }
607
+
486
608
  // It includes every events by default.
487
609
  _skipEvent(name) {
488
- const includes = this.filters.includeEvents;
489
- const excludes = this.filters.excludeEvents;
490
-
491
- let included = (includes === undefined) ||
492
- (Array.isArray(includes) && (includes.indexOf(name) > -1));
493
- let excluded = Array.isArray(excludes) && (excludes.indexOf(name) > -1);
494
-
495
- return excluded || !included;
610
+ if (this._excludeEventsSet.has(name)) {
611
+ return true;
612
+ }
613
+ return this._includeEventsSet !== undefined &&
614
+ !this._includeEventsSet.has(name);
496
615
  }
497
616
 
498
617
  // It doesn't skip any schema by default.
499
618
  _skipSchema(database, table) {
500
- const includes = this.filters.includeSchema;
501
- const excludes = this.filters.excludeSchema || {};
502
-
503
- let included = (includes === undefined) ||
504
- (
505
- (database in includes) &&
506
- (
507
- includes[database] === true ||
508
- (
509
- Array.isArray(includes[database]) &&
510
- includes[database].indexOf(table) > -1
511
- )
512
- )
513
- );
514
- let excluded = (database in excludes) &&
515
- (
516
- excludes[database] === true ||
517
- (
518
- Array.isArray(excludes[database]) &&
519
- excludes[database].indexOf(table) > -1
520
- )
521
- );
619
+ const excludeEntry = this._excludeSchemaMap.get(database);
620
+ if (excludeEntry === true ||
621
+ (excludeEntry instanceof Set && excludeEntry.has(table))) {
622
+ return true;
623
+ }
522
624
 
523
- return excluded || !included;
625
+ if (this._includeSchemaMap === undefined) {
626
+ return false;
627
+ }
628
+ const includeEntry = this._includeSchemaMap.get(database);
629
+ const included = includeEntry === true ||
630
+ (includeEntry instanceof Set && includeEntry.has(table));
631
+ return !included;
524
632
  }
525
633
  }
526
634
 
@@ -364,6 +364,43 @@ class Unknown extends BinlogEvent {
364
364
  }
365
365
  }
366
366
 
367
+ /* MySQL 8.0.20+ wraps whole transactions in a single compressed event when
368
+ * binlog_transaction_compression=ON. The embedded row events are
369
+ * zstd-compressed and zongji cannot decode them, so the row changes they
370
+ * carry would be silently lost. The first occurrence emits an error via
371
+ * ZongJi#_warnUnsupportedEvent (see unsupportedReason below).
372
+ */
373
+ class TransactionPayload extends BinlogEvent {
374
+ constructor(parser, options, zongji) {
375
+ super(parser, options);
376
+ skipEventRemainder(parser, zongji);
377
+ }
378
+
379
+ static unsupportedReason =
380
+ 'Server sent a TRANSACTION_PAYLOAD_EVENT (binlog_transaction_compression=ON). ' +
381
+ 'zongji cannot decode compressed transactions, so their row events are NOT ' +
382
+ 'being emitted. Set binlog_transaction_compression=OFF on the server to ' +
383
+ 'receive these changes.';
384
+ }
385
+
386
+ /* MySQL 8.0+ emits partial JSON row updates when
387
+ * binlog_row_value_options=PARTIAL_JSON. The JSON diff format is not
388
+ * decodable by zongji, so these updates would be silently lost. The first
389
+ * occurrence emits an error via ZongJi#_warnUnsupportedEvent.
390
+ */
391
+ class PartialUpdateRows extends BinlogEvent {
392
+ constructor(parser, options, zongji) {
393
+ super(parser, options);
394
+ skipEventRemainder(parser, zongji);
395
+ }
396
+
397
+ static unsupportedReason =
398
+ 'Server sent a PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=PARTIAL_JSON). ' +
399
+ 'zongji cannot decode partial JSON updates, so these UPDATE row events are ' +
400
+ 'NOT being emitted. Clear binlog_row_value_options on the server to receive ' +
401
+ 'these changes.';
402
+ }
403
+
367
404
  export {
368
405
  BinlogEvent,
369
406
  Rotate,
@@ -375,5 +412,7 @@ export {
375
412
  Gtid,
376
413
  AnonymousGtid,
377
414
  PreviousGtids,
415
+ TransactionPayload,
416
+ PartialUpdateRows,
378
417
  Unknown
379
418
  };
package/lib/code_map.js CHANGED
@@ -37,7 +37,13 @@ const CodeEvent = [
37
37
  'DELETE_ROWS_EVENT_V2',
38
38
  'GTID_LOG_EVENT',
39
39
  'ANONYMOUS_GTID_LOG_EVENT',
40
- 'PREVIOUS_GTIDS_LOG_EVENT'
40
+ 'PREVIOUS_GTIDS_LOG_EVENT',
41
+ 'TRANSACTION_CONTEXT_EVENT',
42
+ 'VIEW_CHANGE_EVENT',
43
+ 'XA_PREPARE_LOG_EVENT',
44
+ 'PARTIAL_UPDATE_ROWS_EVENT',
45
+ 'TRANSACTION_PAYLOAD_EVENT',
46
+ 'HEARTBEAT_LOG_EVENT_V2'
41
47
  ];
42
48
 
43
49
  const EventClass = {
@@ -50,6 +56,8 @@ const EventClass = {
50
56
  GTID_LOG_EVENT: events.Gtid,
51
57
  ANONYMOUS_GTID_LOG_EVENT: events.AnonymousGtid,
52
58
  PREVIOUS_GTIDS_LOG_EVENT: events.PreviousGtids,
59
+ TRANSACTION_PAYLOAD_EVENT: events.TransactionPayload,
60
+ PARTIAL_UPDATE_ROWS_EVENT: events.PartialUpdateRows,
53
61
 
54
62
  TABLE_MAP_EVENT: events.TableMap,
55
63
  DELETE_ROWS_EVENT_V1: rowsEvents.DeleteRows,
package/lib/common.js CHANGED
@@ -1,7 +1,6 @@
1
1
  import iconv from 'iconv-lite';
2
- import decodeJson from './json_decode.js';
2
+ import decodeJson, { convertBigInts, jsonToText } from './json_decode.js';
3
3
  import * as dtDecode from './datetime_decode.js';
4
- import bigInt from 'big-integer';
5
4
 
6
5
  export const MysqlTypes = {
7
6
  DECIMAL: 0,
@@ -40,7 +39,27 @@ export const MysqlTypes = {
40
39
  };
41
40
 
42
41
  const TWO_TO_POWER_THIRTY_TWO = Math.pow(2, 32);
43
- const TWO_TO_POWER_SIXTY_THREE = '9223372036854775808'; // Math.pow(2, 63) or 1 << 63
42
+ const TWO_TO_POWER_SIXTY_THREE = 2n ** 63n;
43
+ const TWO_TO_POWER_SIXTY_FOUR = 2n ** 64n;
44
+ const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
45
+ const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
46
+
47
+ // Combine two unsigned 32-bit halves into an exact unsigned 64-bit value.
48
+ // Returns a Number when at or below 2^53 - 1, otherwise an exact string.
49
+ export function unsignedInt64ToNumberOrString(high, low) {
50
+ const value = (BigInt(high) << 32n) | BigInt(low);
51
+ return value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
52
+ }
53
+
54
+ // Combine two unsigned 32-bit halves into an exact signed (two's
55
+ // complement) 64-bit value. Returns a Number when within the safe integer
56
+ // range, otherwise an exact string.
57
+ export function signedInt64ToNumberOrString(high, low) {
58
+ const value = BigInt.asIntN(64, (BigInt(high) << 32n) | BigInt(low));
59
+ return (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) ?
60
+ Number(value) : value.toString();
61
+ }
62
+
44
63
  // This function will return a Number
45
64
  // if the result < Math.MAX_SAFE_INTEGER or result > Math.MIN_SAFE_INTEGER,
46
65
  // otherwise, will return a string.
@@ -48,11 +67,11 @@ export const parseUInt64 = function(parser) {
48
67
  const low = parser.parseUnsignedNumber(4);
49
68
  const high = parser.parseUnsignedNumber(4);
50
69
 
51
- if (high >>> 21) { // using bigint here
52
- return bigInt(TWO_TO_POWER_THIRTY_TWO).multiply(high).add(low).toString();
70
+ if (high >>> 21) { // value is 2^53 or more, exceeds safe Number range
71
+ return unsignedInt64ToNumberOrString(high, low);
53
72
  }
54
73
 
55
- return (high * Math.pow(2,32)) + low;
74
+ return (high * TWO_TO_POWER_THIRTY_TWO) + low;
56
75
  };
57
76
 
58
77
  export function parseUInt48(parser) {
@@ -115,12 +134,6 @@ export const parseIEEE754Float = function(high, low) {
115
134
  }
116
135
  };
117
136
 
118
- export function getUInt32Value(input) {
119
- // Last bit is not sign, it is part of value!
120
- if (input & (1 << 31)) return Math.pow(2, 31) + (input & ((1 << 31) -1));
121
- else return input;
122
- };
123
-
124
137
  const parseAnyInt = function(parser, column, columnSchema) {
125
138
  let result, size;
126
139
  switch (column.type) {
@@ -151,16 +164,13 @@ const parseAnyInt = function(parser, column, columnSchema) {
151
164
  // Flip bits on negative signed integer
152
165
  if (!int64 && (result & (1 << (length - 1)))) {
153
166
  result = ((result ^ (Math.pow(2, length) - 1)) * -1) - 1;
154
- } else if (int64 && bigInt(result).greaterOrEquals(bigInt(TWO_TO_POWER_SIXTY_THREE))) {
155
- const Max64BitNumber = bigInt('18446744073709551615'); // 2^64 - 1
156
- result = bigInt(result).xor(Max64BitNumber).add(1).multiply(-1);
157
- // Javascript integers only support 2^53, if not within the range, return a String
158
- if (result.greater(Number.MAX_SAFE_INTEGER) || result.lesser(Number.MIN_SAFE_INTEGER)) {
159
- result = result.toString();
160
- }
161
- // Otherwise return a Number
162
- else {
163
- result = result.toJSNumber();
167
+ } else if (int64) {
168
+ const unsigned = BigInt(result);
169
+ if (unsigned >= TWO_TO_POWER_SIXTY_THREE) {
170
+ const signed = unsigned - TWO_TO_POWER_SIXTY_FOUR;
171
+ // Javascript Numbers only cover +-(2^53 - 1); return an exact
172
+ // string outside that range
173
+ result = signed >= MIN_SAFE_BIGINT ? Number(signed) : signed.toString();
164
174
  }
165
175
  }
166
176
  }
@@ -185,7 +195,11 @@ const readIntBE = function(buf, offset, length, noAssert) {
185
195
  // https://github.com/jeremycole/mysql_binlog/blob/master/lib/mysql_binlog/binlog_field_parser.rb
186
196
  // Some more information about DECIMAL types:
187
197
  // http://dev.mysql.com/doc/refman/5.5/en/precision-math-decimal-characteristics.html
188
- const parseNewDecimal = function(parser, column) {
198
+ // Returns the exact decimal value as a string (matching mysql2's default
199
+ // treatment of DECIMAL columns), or as a Number if the connection was
200
+ // configured with the mysql2 `decimalNumbers` option (which may lose
201
+ // precision beyond 15 significant digits).
202
+ const parseNewDecimal = function(parser, column, zongji) {
189
203
  // Constants of format
190
204
  const digitsPerInteger = 9;
191
205
  const compressedBytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4];
@@ -197,38 +211,37 @@ const parseNewDecimal = function(parser, column) {
197
211
  const compIntegral = integral - (uncompIntegral * digitsPerInteger);
198
212
  const compFractional = scale - (uncompFractional * digitsPerInteger);
199
213
 
200
- // Grab buffer portion
214
+ // Copy buffer portion: the sign bit is flipped in place below and the
215
+ // underlying packet buffer must not be mutated
201
216
  const size = (uncompIntegral * 4) + compressedBytes[compIntegral] +
202
217
  (uncompFractional * 4) + compressedBytes[compFractional];
203
- const buffer = parser._buffer.slice(parser._offset, parser._offset + size);
218
+ const buffer = Buffer.from(
219
+ parser._buffer.subarray(parser._offset, parser._offset + size));
204
220
  parser._offset += size; // Move binlog parser position forward
205
221
 
206
- let str, mask;
207
222
  let pos = 0;
208
223
  const isPositive = (buffer.readUInt8(0) & (1 << 7)) === 128;
209
224
  buffer.writeUInt8(buffer.readUInt8(0) ^ (1 << 7), 0);
210
- if (isPositive) {
211
- // Positive number
212
- str = '';
213
- mask = 0;
214
- } else {
215
- // Negative number
216
- str = '-';
217
- mask = -1;
218
- }
225
+ // Negative numbers are stored with all bits flipped
226
+ const mask = isPositive ? 0 : -1;
219
227
 
220
228
  // Build integer digits
229
+ let intDigits = '';
221
230
  const compIntegralSize = compressedBytes[compIntegral];
222
231
  if (compIntegralSize > 0) {
223
- str += (readIntBE(buffer, 0, compIntegralSize) ^ mask).toString(10);
232
+ intDigits += (readIntBE(buffer, 0, compIntegralSize) ^ mask).toString(10);
224
233
  pos += compIntegralSize;
225
234
  }
226
235
 
227
236
  for (let i = 0; i < uncompIntegral; i++) {
228
- str += zeroPad((buffer.readInt32BE(pos) ^ mask).toString(10), 9);
237
+ intDigits += zeroPad((buffer.readInt32BE(pos) ^ mask).toString(10), 9);
229
238
  pos += 4;
230
239
  }
231
240
 
241
+ // Strip leading zeros, keeping at least one digit
242
+ intDigits = intDigits.replace(/^0+(?=\d)/, '');
243
+ if (intDigits === '') intDigits = '0';
244
+
232
245
  // Build fractional digits
233
246
  let fractionDigits = '';
234
247
 
@@ -242,10 +255,13 @@ const parseNewDecimal = function(parser, column) {
242
255
  fractionDigits += zeroPad((readIntBE(buffer, pos, compFractionalSize) ^ mask).toString(10), compFractional);
243
256
  }
244
257
 
245
- // Fractional digits may have leading zeros
246
- str += '.' + fractionDigits;
258
+ let str = (isPositive ? '' : '-') + intDigits;
259
+ if (scale > 0) {
260
+ str += '.' + fractionDigits;
261
+ }
247
262
 
248
- return parseFloat(str);
263
+ const config = zongji && zongji.connection && zongji.connection.config;
264
+ return config && config.decimalNumbers ? parseFloat(str) : str;
249
265
  };
250
266
 
251
267
  // Did not work in place. Function cribbed from lines 311-363 of
@@ -380,7 +396,7 @@ export function readMysqlValue(
380
396
  result = parseIEEE754Float(high, low);
381
397
  break;
382
398
  case MysqlTypes.NEWDECIMAL:
383
- result = parseNewDecimal(parser, column);
399
+ result = parseNewDecimal(parser, column, zongji);
384
400
  break;
385
401
  case MysqlTypes.SET:
386
402
  if (column.metadata.size === 8) {
@@ -454,12 +470,24 @@ export function readMysqlValue(
454
470
  }
455
471
  }
456
472
  break;
457
- case MysqlTypes.JSON:
473
+ case MysqlTypes.JSON: {
458
474
  lengthSize = column.metadata['length_size'];
459
475
  size = parser.parseUnsignedNumber(lengthSize);
460
476
  buffer = parser.parseBuffer(size);
461
- result = buffer.length === 0 ? null : decodeJson(buffer);
477
+ if (buffer.length === 0) {
478
+ result = null;
479
+ } else {
480
+ // Match mysql2 semantics: parsed JavaScript value by default,
481
+ // raw JSON text with the jsonStrings connection option. 64-bit
482
+ // integers stay exact: raw numerals in text mode, Numbers within
483
+ // the safe range (exact strings beyond it) in parsed mode.
484
+ const decoded = decodeJson(buffer);
485
+ const config = zongji && zongji.connection && zongji.connection.config;
486
+ result = config && config.jsonStrings ?
487
+ jsonToText(decoded) : convertBigInts(decoded);
488
+ }
462
489
  break;
490
+ }
463
491
  case MysqlTypes.GEOMETRY:
464
492
  lengthSize = column.metadata['length_size'];
465
493
  size = parser.parseUnsignedNumber(lengthSize);