@vlasky/zongji 0.6.1 → 0.7.0

package/lib/json_decode.js

@@ -21,11 +21,62 @@ const JSONB_LITERALS = [ null, true, false ];
 // node-mysql prefixes binary string values
 const VAR_STRING_PREFIX = 'base64:type253:';
 
+// Decodes a JSONB binary buffer into JavaScript values (object, array,
+// string, number, boolean, BigInt for 64-bit integers, or null). The
+// caller converts BigInts and decides whether to serialise back to a JSON
+// string (mysql2 `jsonStrings` option) via the helpers below.
 export default function decodeJson(input) {
-  // Value must be JSON string to match node-mysql results
-  // Related to this node-mysql PR:
-  // https://github.com/felixge/node-mysql/pull/1299
-  return JSON.stringify(parseBinaryBuffer(input));
+  return parseBinaryBuffer(input);
+}
+
+const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
+
+// For parsed-value output: BigInts become Numbers when within the safe
+// integer range, exact strings otherwise. Mutates objects/arrays in place
+// (they were freshly created by parseBinaryBuffer).
+export function convertBigInts(value) {
+  if (typeof value === 'bigint') {
+    return (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) ?
+      Number(value) : value.toString();
+  }
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i++) {
+      value[i] = convertBigInts(value[i]);
+    }
+    return value;
+  }
+  if (value !== null && typeof value === 'object') {
+    for (const key of Object.keys(value)) {
+      value[key] = convertBigInts(value[key]);
+    }
+    return value;
+  }
+  return value;
+}
+
+// For jsonStrings output: serialise to JSON text with 64-bit integers as
+// raw numerals (JSON.stringify cannot represent BigInt), so the text
+// matches what the server would print for the column.
+export function jsonToText(value) {
+  if (value === null) {
+    return 'null';
+  }
+  switch (typeof value) {
+    case 'bigint':
+    case 'number':
+    case 'boolean':
+      return String(value);
+    case 'string':
+      return JSON.stringify(value);
+    default:
+      if (Array.isArray(value)) {
+        return '[' + value.map(jsonToText).join(', ') + ']';
+      }
+      return '{' + Object.keys(value).map(key =>
+        JSON.stringify(key) + ': ' + jsonToText(value[key])
+      ).join(', ') + '}';
+  }
 }
 
 /*
@@ -112,25 +163,17 @@ function parseBinaryBuffer(input, offset, parentValueOffset, readUInt, isLarge)
       result = isLarge ? input.readUInt32LE(offset + 1) : input.readUInt32LE(valueOffset + 1);
       break;
     case JSONB_TYPE_INT64: {
-      let low = input.readUInt32LE(valueOffset + 1);
-      let high = input.readUInt32LE(valueOffset + 5);
-      if (high & (1 << 31)) {
-        // Javascript integers only support 2^53 not 2^64, must trim bits!
-        // 64-53 = 11, 32-11 = 21, so grab first 21 bits of high word only
-        const mask = Math.pow(2, 32) - 1;
-        high = common.sliceBits(high ^ mask, 0, 21);
-        low = low ^ mask;
-        result =
-          ((high * Math.pow(2, 32)) * - 1) - common.getUInt32Value(low) - 1;
-      } else {
-        result = (high * Math.pow(2,32)) + low;
-      }
+      const low = input.readUInt32LE(valueOffset + 1);
+      const high = input.readUInt32LE(valueOffset + 5);
+      // Exact; converted to Number / string / raw JSON numeral by the
+      // consumers below
+      result = BigInt.asIntN(64, (BigInt(high) << 32n) | BigInt(low));
       break;
     }
     case JSONB_TYPE_UINT64: {
       const low = input.readUInt32LE(valueOffset + 1);
       const high = input.readUInt32LE(valueOffset + 5);
-      result = (high * Math.pow(2,32)) + low;
+      result = (BigInt(high) << 32n) | BigInt(low);
       break;
     }
     case JSONB_TYPE_OPAQUE: {

package/lib/packet/binlog.js

@@ -32,6 +32,13 @@ export default function initBinlogPacketClass(zongji) {
       const EventClass = getEventClass(eventType);
       this.eventName = EventClass.name;
 
+      // Warn (once per instance) about event types that carry row changes
+      // zongji cannot decode. This must happen before event filtering so
+      // the data loss is loud even when the type is not included.
+      if (EventClass.unsupportedReason) {
+        zongji._warnUnsupportedEvent(EventClass);
+      }
+
       yield;
 
       try {

package/lib/reader.js

@@ -1,145 +1,6 @@
-// Constants for variable length encoded binary
-export const NULL_COLUMN = 251;
-export const UNSIGNED_CHAR_COLUMN = 251;
-export const UNSIGNED_SHORT_COLUMN = 252;
-export const UNSIGNED_INT24_COLUMN = 253;
-export const UNSIGNED_INT64_COLUMN = 254;
-
-const padWith = function(val, length) {
-  const bits = val.split('');
-  if (bits.length < length) {
-    const left = length - bits.length;
-    for (let j = left - 1; j >= 0; j--) {
-      bits.unshift('0');
-    }
-    val = bits.join('');
-  }
-
-  return val;
-};
-
-class BufferReader {
-  constructor(buffer) {
-    this.buffer = buffer;
-    this.position = 0;
-  }
-
-  readUInt8() {
-    const pos = this.position;
-    this.position += 1;
-
-    return this.buffer.readUInt8(pos);
-  }
-
-  readUInt16() {
-    const pos = this.position;
-    this.position += 2;
-
-    return this.buffer.readUInt16LE(pos);
-  }
-
-  readUInt32() {
-    const pos = this.position;
-    this.position += 4;
-
-    return this.buffer.readUInt32LE(pos);
-  }
-
-  readUInt24() {
-    const low = this.readUInt16();
-    const high = this.readUInt8();
-    return (high << 16) + low;
-  }
-
-  readUInt64() {
-    const pos = this.position;
-    this.position += 8;
-
-    // from http://stackoverflow.com/questions/17687307/convert-a-64bit-little-endian-integer-to-number
-    return this.buffer.readInt32LE(pos) +
-      0x100000000 * this.buffer.readUInt32LE(pos + 4);
-  }
-
-  readString() {
-    const strBuf = this.buffer.slice(this.position);
-    this.position = this.buffer.length;
-
-    return strBuf.toString('ascii');
-  }
-
-  readStringInBytes(length) {
-    const strBuf = this.buffer.slice(this.position, this.position + length);
-    this.position += length;
-
-    return strBuf.toString('ascii');
-  }
-
-  readHexInBytes(length) {
-    const buf = this.buffer.slice(this.position, this.position + length);
-    this.position += length;
-
-    return buf.toString('hex');
-  }
-
-  readBytesArray(length) {
-    const result = [];
-    const hexString = this.readHexInBytes(length);
-    for (let i = 0; i < hexString.length; i = i + 2) {
-      result.push(parseInt(hexString.substr(i, 2), 16));
-    }
-    return result;
-  }
-
-  // Read a variable-length "Length Coded Binary" integer. This is derived
-  // from the MySQL protocol, and re-used in the binary log format. This
-  // format uses the first byte to alternately store the actual value for
-  // integer values <= 250, or to encode the number of following bytes
-  // used to store the actual value, which can be 2, 3, or 8. It also
-  // includes support for SQL NULL as a special case.
-  readVariant() {
-    let result = null;
-    const firstByte = this.readUInt8();
-
-    if (firstByte < UNSIGNED_CHAR_COLUMN) {
-      result = firstByte;
-    } else if (firstByte === NULL_COLUMN) {
-      result = null;
-    } else if (firstByte === UNSIGNED_SHORT_COLUMN) {
-      result = this.readUInt16();
-    } else if (firstByte === UNSIGNED_INT24_COLUMN) {
-      result = this.readUInt24();
-    } else if (firstByte === UNSIGNED_INT64_COLUMN) {
-      result = this.readUInt64();
-    } else {
-      throw new Error('Invalid variable-length integer');
-    }
-
-    return result;
-  }
-
-  // Read an arbitrary-length bitmap, provided its length.
-  // Returns an array of true/false values.
-  readBitArray(length) {
-    const size = Math.floor((length + 7) / 8);
-
-    const bytes = [];
-    for (let i = size - 1; i >= 0; i--) {
-      bytes.unshift(this.readUInt8());
-    }
-
-    const bitmap = [];
-    const bitmapStr = bytes.map(function(aByte) {
-      return padWith(aByte.toString(2), 8);
-    }).join('');
-
-    for (let k = bitmapStr.length - 1; k >= 0; k--) {
-      bitmap.push(bitmapStr[k] === '1');
-    }
-
-    return bitmap.slice(0, length);
-  }
-}
-
+// Minimal parser over a mysql2 packet (or a bare buffer via append()),
+// providing the subset of the node-mysql Parser interface that the binlog
+// event classes consume.
 class Parser {
   constructor(packet) {
     this.packet = null;
@@ -244,4 +105,4 @@ class Parser {
   }
 }
 
-export { BufferReader, Parser };
+export { Parser };

package/lib/sequence/binlog.js

@@ -1,5 +1,5 @@
 import { EventEmitter } from 'events';
-import { initBinlogPacketClass } from '../packet/index.js';
+import initBinlogPacketClass from '../packet/binlog.js';
 import { Parser } from '../reader.js';
 
 const BINLOG_DUMP_COMMAND = 0x12;
@@ -137,16 +137,38 @@ export default function initBinlogClass(zongji) {
         const eventName = binlogPacket.eventName
           ? binlogPacket.eventName.toLowerCase()
           : 'unknown';
+
+        let event;
+        let error;
+
+        // Track the current transaction GTID before event filtering, so
+        // subsequent events can carry event.gtid even when 'gtid' events
+        // themselves are excluded by includeEvents
+        const isGtidEvent =
+          eventName === 'gtid' || eventName === 'anonymousgtid';
+        if (isGtidEvent) {
+          try {
+            event = binlogPacket.getEvent();
+            // Anonymous transactions have no usable GTID; do not let a
+            // previous transaction's GTID leak onto their events
+            zongji._currentGtid =
+              eventName === 'gtid' ? event.gtid : undefined;
+          } catch (err) {
+            zongji._currentGtid = undefined;
+            error = err;
+          }
+        }
+
         if (zongji._skipEvent(eventName)) {
           return Binlog.prototype.binlogData;
         }
 
-        let event;
-        let error;
-        try {
-          event = binlogPacket.getEvent();
-        } catch (err) {
-          error = err;
+        if (!isGtidEvent && !error) {
+          try {
+            event = binlogPacket.getEvent();
+          } catch (err) {
+            error = err;
+          }
         }
         this._callback.call(this, error, event);
       }

package/package.json

@@ -1,16 +1,20 @@
 {
   "name": "@vlasky/zongji",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "MySQL binlog-based change data capture (CDC) for Node.js",
   "type": "module",
   "main": "index.js",
   "types": "index.d.ts",
-  "directories": {
-    "test": "test"
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    }
   },
   "scripts": {
     "pretest": "bash scripts/start-mysql.sh",
     "test": "tap --bail --disable-coverage --jobs=1 test/*.js",
+    "test:only": "tap --bail --disable-coverage --jobs=1 test/*.js",
     "lint": "eslint .",
     "checkjs": "tsc --project jsconfig.json"
   },
@@ -42,8 +46,7 @@
     "tap": "^21.5.0"
   },
   "dependencies": {
-    "big-integer": "1.6.52",
     "iconv-lite": "0.7.2",
-    "mysql2": "^3.17.1"
+    "mysql2": "^3.22.5"
   }
 }

package/.travis.yml

@@ -1,15 +0,0 @@
-language: node_js
-sudo: required
-dist: trusty
-node_js:
-  - "8"
-  - "12"
-services:
-  - docker
-before_script:
-  - npm run lint
-  - docker run -p 3355:3306 --name mysql-55 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.5 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
-  - docker run -p 3356:3306 --name mysql-56 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.6 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
-  - docker run -p 3357:3306 --name mysql-57 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.7 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
-script:
-  - ./test/travis/runner.sh

package/docker-compose.yml

@@ -1,33 +0,0 @@
----
-services:
-  mysql57:
-    image: mysql:5.7
-    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
-    networks:
-      default:
-        aliases:
-          - mysql57
-    environment:
-      MYSQL_ROOT_PASSWORD: secret
-
-  mysql80:
-    image: mysql:8.0
-    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
-    networks:
-      default:
-        aliases:
-          - mysql80
-    environment:
-      MYSQL_ROOT_PASSWORD: secret
-
-  mysql84:
-    image: mysql:8.4
-    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
-    ports:
-      - "3306:3306"
-    networks:
-      default:
-        aliases:
-          - mysql84
-    environment:
-      MYSQL_ROOT_PASSWORD: secret

package/docker-test.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-MYSQL_HOSTS="mysql57 mysql80 mysql84"
-
-for hostname in ${MYSQL_HOSTS}; do
-  echo $hostname + node 18
-  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:18 npm test
-  echo $hostname + node 20
-  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:20 npm test
-  echo $hostname + node 22
-  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:22 npm test
-done

package/eslint.config.js

@@ -1,35 +0,0 @@
-import js from '@eslint/js';
-
-export default [
-  { ignores: ['.tap/'] },
-  js.configs.recommended,
-  {
-    languageOptions: {
-      ecmaVersion: 2022,
-      sourceType: 'module',
-      globals: {
-        Buffer: 'readonly',
-        clearImmediate: 'readonly',
-        clearInterval: 'readonly',
-        clearTimeout: 'readonly',
-        console: 'readonly',
-        process: 'readonly',
-        setImmediate: 'readonly',
-        setInterval: 'readonly',
-        setTimeout: 'readonly',
-      },
-    },
-    rules: {
-      'comma-dangle': ['warn', 'only-multiline'],
-      'eol-last': ['error'],
-      'keyword-spacing': ['error', { before: true }],
-      'no-console': 'off',
-      'no-trailing-spaces': ['error', { skipBlankLines: true }],
-      'no-unused-vars': 'warn',
-      'no-var': 'warn',
-      'quotes': ['warn', 'single', 'avoid-escape'],
-      'semi': ['error', 'always'],
-      'space-before-blocks': 'error',
-    },
-  },
-];

package/jsconfig.json

@@ -1,15 +0,0 @@
-{
-  "compilerOptions": {
-    "checkJs": true,
-    "allowJs": true,
-    "noEmit": true,
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "skipDefaultLibCheck": true,
-    "maxNodeModuleJsDepth": 0,
-    "types": ["node"]
-  },
-  "include": ["index.js", "index.d.ts", "lib/**/*.js"],
-  "exclude": ["node_modules", ".tap"]
-}

package/lib/packet/combinlog.js

@@ -1,27 +0,0 @@
-class ComBinlog {
-  constructor({ serverId, nonBlock, filename, position }) {
-    this.command = 0x12;
-    this.position = position || 4;
-
-    // will send eof package if there is no more binlog event
-    // https://dev.mysql.com/doc/internals/en/com-binlog-dump.html#binlog-dump-non-block
-    this.flags = nonBlock ? 1 : 0;
-
-    this.serverId = serverId || 1;
-    this.filename = filename || '';
-  }
-
-  write(writer) {
-    writer.writeUnsignedNumber(1, this.command);
-    writer.writeUnsignedNumber(4, this.position);
-    writer.writeUnsignedNumber(2, this.flags);
-    writer.writeUnsignedNumber(4, this.serverId);
-    writer.writeNullTerminatedString(this.filename);
-  }
-
-  parse() {
-    throw new Error('should never be called here');
-  }
-}
-
-export default ComBinlog;

package/lib/packet/index.js

@@ -1,66 +0,0 @@
-import ComBinlog from './combinlog.js';
-import initBinlogPacketClass from './binlog.js';
-
-// from npm package mysql
-class EofPacket {
-  constructor(options = {}) {
-    this.fieldCount = undefined;
-    this.warningCount = options.warningCount;
-    this.serverStatus = options.serverStatus;
-    this.protocol41 = options.protocol41;
-  }
-
-  parse(parser) {
-    this.fieldCount = parser.parseUnsignedNumber(1);
-    if (this.protocol41) {
-      this.warningCount = parser.parseUnsignedNumber(2);
-      this.serverStatus = parser.parseUnsignedNumber(2);
-    }
-  }
-
-  write(writer) {
-    writer.writeUnsignedNumber(1, 0xfe);
-    if (this.protocol41) {
-      writer.writeUnsignedNumber(2, this.warningCount);
-      writer.writeUnsignedNumber(2, this.serverStatus);
-    }
-  }
-}
-
-// from npm package mysql
-class ErrorPacket {
-  constructor(options = {}) {
-    this.fieldCount = options.fieldCount;
-    this.errno = options.errno;
-    this.sqlStateMarker = options.sqlStateMarker;
-    this.sqlState = options.sqlState;
-    this.message = options.message;
-  }
-
-  parse(parser) {
-    this.fieldCount = parser.parseUnsignedNumber(1);
-    this.errno = parser.parseUnsignedNumber(2);
-
-    // sqlStateMarker ('#' = 0x23) indicates error packet format
-    if (parser.peak() === 0x23) {
-      this.sqlStateMarker = parser.parseString(1);
-      this.sqlState = parser.parseString(5);
-    }
-
-    this.message = parser.parsePacketTerminatedString();
-  }
-
-  write(writer) {
-    writer.writeUnsignedNumber(1, 0xff);
-    writer.writeUnsignedNumber(2, this.errno);
-
-    if (this.sqlStateMarker) {
-      writer.writeString(this.sqlStateMarker);
-      writer.writeString(this.sqlState);
-    }
-
-    writer.writeString(this.message);
-  }
-}
-
-export { EofPacket, ErrorPacket, ComBinlog, initBinlogPacketClass };

package/scripts/start-mysql.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-# Start MySQL Docker containers for testing
-# Skips MySQL 5.7 on ARM64 (Apple Silicon) as no image is available
-
-if [ "$(uname -m)" = "arm64" ]; then
-  SERVICES="mysql80 mysql84"
-else
-  SERVICES="mysql57 mysql80 mysql84"
-fi
-
-docker-compose up -d $SERVICES
-
-# Wait for MySQL to be ready
-for service in $SERVICES; do
-  container="zongji-${service}-1"
-  echo -n "Waiting for ${service}..."
-  for i in $(seq 1 30); do
-    if docker exec "$container" mysqladmin ping -u root -psecret --silent 2>/dev/null; then
-      echo " ready"
-      break
-    fi
-    if [ "$i" -eq 30 ]; then
-      echo " timed out"
-      exit 1
-    fi
-    sleep 2
-  done
-done