@vlasky/zongji 0.7.1 → 0.8.0

package/lib/gtid_set.js

@@ -0,0 +1,237 @@
+// A MySQL GTID set: which transactions (per source) are already
+// executed. String form matches @@gtid_executed / Previous_gtids, e.g.
+//   3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5:11-18,
+//   2C256447-3F0D-431B-9A25-BDDF1F1F6EF6:1-27
+// MySQL 8.3+ GTIDs may carry a tag; a (uuid, tag) pair is its own
+// source, printed merged into the uuid's block after the untagged
+// intervals: 'uuid:1-5:tag_a:1:tag_b:1' (tags lowercase, sorted).
+// Intervals are stored internally as [start, end] inclusive, kept sorted
+// and coalesced. GNOs are JavaScript Numbers: MySQL allows up to 2^63-1,
+// but real-world transaction counts sit comfortably inside 2^53.
+class GtidSet {
+  constructor() {
+    // uuid (lowercase, dashed) -> Map of tag ('' = untagged) ->
+    // array of [start, end] inclusive
+    this._sids = new Map();
+  }
+
+  // Accepts '', 'uuid:1-5', 'uuid:1-5:8,uuid2:3',
+  // 'uuid:1-5:tag_a:1:tag_b:2-3' (whitespace tolerated). Within a uuid
+  // block a non-numeric item is a tag naming the source of the interval
+  // items that follow it, per the Gtid_set::add_gtid_text grammar.
+  static parse(text) {
+    const set = new GtidSet();
+    if (!text) {
+      return set;
+    }
+    for (const entry of String(text).split(',')) {
+      const parts = entry.trim().split(':');
+      if (parts.length < 2) {
+        throw new Error(`Invalid GTID set entry: '${entry.trim()}'`);
+      }
+      const uuid = normaliseUuid(parts[0]);
+      let tag = '';
+      let intervals = 0;
+      // A tag names the interval items that follow it, so a tag section
+      // with none is malformed (the leading untagged section may be
+      // empty: 'uuid:tag_a:1' has no untagged intervals)
+      let sectionEmpty = false;
+      for (const item of parts.slice(1)) {
+        const trimmed = item.trim();
+        if (TAG_REGEX.test(trimmed)) {
+          if (sectionEmpty) {
+            throw new Error(`GTID tag without intervals: '${entry.trim()}'`);
+          }
+          tag = trimmed.toLowerCase();
+          sectionEmpty = true;
+          continue;
+        }
+        const bounds = trimmed.split('-');
+        const start = parsePositiveInt(bounds[0], entry);
+        const end = bounds.length > 1 ?
+          parsePositiveInt(bounds[1], entry) : start;
+        if (bounds.length > 2 || end < start) {
+          throw new Error(`Invalid GTID interval: '${entry.trim()}'`);
+        }
+        set._addInterval(uuid, tag, start, end);
+        sectionEmpty = false;
+        intervals++;
+      }
+      if (intervals === 0 || sectionEmpty) {
+        throw new Error(`Invalid GTID set entry: '${entry.trim()}'`);
+      }
+    }
+    return set;
+  }
+
+  // Adds a single transaction from a 'uuid:gno' or 'uuid:tag:gno' string
+  add(gtid) {
+    const { uuid, tag, gno } = parseSingleGtid(gtid);
+    this._addInterval(uuid, tag, gno, gno);
+  }
+
+  // Whether a single transaction ('uuid:gno' or 'uuid:tag:gno', e.g. an
+  // event.gtid) is already contained in the set. null/undefined (an
+  // anonymous transaction's event.gtid) is never contained; a malformed
+  // string throws, as it would corrupt rather than answer.
+  contains(gtid) {
+    if (gtid === undefined || gtid === null) {
+      return false;
+    }
+    const { uuid, tag, gno } = parseSingleGtid(gtid);
+    const tags = this._sids.get(uuid);
+    const intervals = tags && tags.get(tag);
+    if (!intervals) {
+      return false;
+    }
+    return intervals.some(([start, end]) => gno >= start && gno <= end);
+  }
+
+  // Adds [start, end] inclusive for a (uuid, tag) source, keeping
+  // intervals sorted and coalesced
+  _addInterval(uuid, tag, start, end) {
+    let tags = this._sids.get(uuid);
+    if (!tags) {
+      tags = new Map();
+      this._sids.set(uuid, tags);
+    }
+    const existing = tags.get(tag) || [];
+    existing.push([start, end]);
+    existing.sort((a, b) => a[0] - b[0]);
+    const merged = [existing[0]];
+    for (let i = 1; i < existing.length; i++) {
+      const last = merged[merged.length - 1];
+      const next = existing[i];
+      if (next[0] <= last[1] + 1) {
+        last[1] = Math.max(last[1], next[1]);
+      } else {
+        merged.push(next);
+      }
+    }
+    tags.set(tag, merged);
+  }
+
+  isEmpty() {
+    return this._sids.size === 0;
+  }
+
+  // Sorted (uuid, then tag, untagged first) flat list of
+  // [uuid, tag, intervals] entries: the order both the text form and
+  // the wire encoding require
+  _sortedEntries() {
+    return [...this._sids.entries()]
+      .sort((a, b) => a[0] < b[0] ? -1 : 1)
+      .flatMap(([uuid, tags]) => [...tags.entries()]
+        .sort((a, b) => a[0] < b[0] ? -1 : 1)
+        .map(([tag, intervals]) => [uuid, tag, intervals]));
+  }
+
+  toString() {
+    let text = '';
+    let lastUuid = null;
+    for (const [uuid, tag, intervals] of this._sortedEntries()) {
+      const ranges = intervals
+        .map(([start, end]) => start === end ? `${start}` : `${start}-${end}`)
+        .join(':');
+      if (uuid === lastUuid) {
+        text += `:${tag}:${ranges}`;
+      } else {
+        text += (text ? ',' : '') + uuid +
+          (tag ? `:${tag}` : '') + `:${ranges}`;
+        lastUuid = uuid;
+      }
+    }
+    return text;
+  }
+
+  // Wire encoding used by COM_BINLOG_DUMP_GTID (and Previous_gtids).
+  // Untagged sets use the classic layout accepted by every server
+  // version: n_sids u64le, then per sid: 16-byte binary UUID,
+  // n_intervals u64le, then per interval: start u64le, end-exclusive
+  // u64le. A set containing any tagged source switches the whole
+  // payload to the tagged format (MySQL 8.3+, sql/rpl_gtid_set.cc
+  // encode_nsids_format): the first u64 becomes
+  // (1 << 56) | (n_entries << 8) | 1, and every entry carries a tag
+  // field (one length byte = length << 1, then the lowercase tag
+  // bytes; 0x00 for untagged) between the UUID and the interval count.
+  // Servers before 8.3 reject the tagged format as malformed.
+  encode() {
+    const entries = this._sortedEntries();
+    const tagged = entries.some(([, tag]) => tag !== '');
+    let length = 8;
+    for (const [, tag, intervals] of entries) {
+      length += 16 + (tagged ? 1 + tag.length : 0) + 8 +
+        intervals.length * 16;
+    }
+    const buffer = Buffer.alloc(length);
+    let offset = 0;
+    const nSidsField = tagged ?
+      (1n << 56n) | (BigInt(entries.length) << 8n) | 1n :
+      BigInt(entries.length);
+    offset = buffer.writeBigUInt64LE(nSidsField, offset);
+    for (const [uuid, tag, intervals] of entries) {
+      offset += Buffer.from(uuid.replace(/-/g, ''), 'hex')
+        .copy(buffer, offset);
+      if (tagged) {
+        offset = buffer.writeUInt8(tag.length << 1, offset);
+        offset += buffer.write(tag, offset, 'ascii');
+      }
+      offset = buffer.writeBigUInt64LE(BigInt(intervals.length), offset);
+      for (const [start, end] of intervals) {
+        offset = buffer.writeBigUInt64LE(BigInt(start), offset);
+        offset = buffer.writeBigUInt64LE(BigInt(end) + 1n, offset);
+      }
+    }
+    return buffer;
+  }
+}
+
+const UUID_REGEX =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+// Tag grammar per libs/mysql/gtid/tag.cpp: a letter or underscore, then
+// letters, digits or underscores, at most 32 characters, case-folded to
+// lowercase. Cannot be confused with an interval (those start with a
+// digit).
+const TAG_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]{0,31}$/;
+
+function parseSingleGtid(gtid) {
+  const parts = String(gtid).split(':');
+  if (parts.length !== 2 && parts.length !== 3) {
+    throw new Error(`Invalid GTID: '${gtid}'`);
+  }
+  const uuid = normaliseUuid(parts[0]);
+  let tag = '';
+  if (parts.length === 3) {
+    const candidate = parts[1].trim();
+    if (!TAG_REGEX.test(candidate)) {
+      throw new Error(`Invalid GTID tag in '${gtid}'`);
+    }
+    tag = candidate.toLowerCase();
+  }
+  const gno = parsePositiveInt(parts[parts.length - 1].trim(), gtid);
+  return { uuid, tag, gno };
+}
+
+function normaliseUuid(text) {
+  const uuid = text.trim().toLowerCase();
+  if (!UUID_REGEX.test(uuid)) {
+    throw new Error(`Invalid GTID source UUID: '${text.trim()}'`);
+  }
+  return uuid;
+}
+
+function parsePositiveInt(text, context) {
+  // Decimal digits only, per the MySQL GTID grammar: Number() alone would
+  // also accept '1e3', '0x10' or '1.0' and silently reinterpret them
+  if (!/^\d+$/.test(text)) {
+    throw new Error(`Invalid GTID number in '${context}': '${text}'`);
+  }
+  const value = Number(text);
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new Error(`Invalid GTID number in '${context}': '${text}'`);
+  }
+  return value;
+}
+
+export { GtidSet };

package/lib/mariadb_gtid.js

@@ -0,0 +1,107 @@
+// A MariaDB GTID position: the last transaction processed per replication
+// domain. String form matches @@gtid_binlog_pos / @@gtid_current_pos /
+// @slave_connect_state, e.g. '0-1-1234,1-3-45'. Unlike MySQL GTID sets
+// there are no intervals: within a domain the binlog order is total, so a
+// single domain-server-sequence triple identifies everything before it,
+// and a position holds at most one entry per domain. The server resumes a
+// client by skipping event groups in each listed domain until it passes
+// the given sequence number from the given server id; domains not listed
+// are streamed from the beginning.
+class MariadbGtidPosition {
+  constructor() {
+    // domainId (number) -> { domainId, serverId, seqNo }
+    this._domains = new Map();
+  }
+
+  // Accepts '' or 'D-S-N[,D-S-N...]' (whitespace tolerated). Rejects two
+  // entries for one domain, as the server would (ER_DUPLICATE_GTID_DOMAIN)
+  static parse(text) {
+    const position = new MariadbGtidPosition();
+    if (!text) {
+      return position;
+    }
+    for (const entry of String(text).split(',')) {
+      const parts = entry.trim().split('-');
+      if (parts.length !== 3) {
+        throw new Error(`Invalid MariaDB GTID: '${entry.trim()}'`);
+      }
+      const domainId = parseUInt32(parts[0], entry);
+      const serverId = parseUInt32(parts[1], entry);
+      const seqNo = parseSeqNo(parts[2], entry);
+      if (position._domains.has(domainId)) {
+        throw new Error(
+          `Duplicate replication domain ${domainId} in GTID position ` +
+          `'${text}'`);
+      }
+      position._domains.set(domainId, { domainId, serverId, seqNo });
+    }
+    return position;
+  }
+
+  // Builds the position implied by a GTID_LIST_EVENT: the event may hold
+  // several entries per domain (one per server id); the last entry of a
+  // domain's run is the most recent, so plain per-domain overwrite in
+  // event order yields the right watermark
+  static fromGtidList(entries) {
+    const position = new MariadbGtidPosition();
+    for (const entry of entries) {
+      position.add(entry);
+    }
+    return position;
+  }
+
+  // Records { domainId, serverId, seqNo } as the last transaction
+  // processed in its domain. Deliberately an overwrite, not a max: after
+  // a failover within a domain the newest transaction can carry a lower
+  // sequence number under a different server id, and "last processed" is
+  // what the server's skip rule keys on
+  add(gtid) {
+    this._domains.set(gtid.domainId, {
+      domainId: gtid.domainId,
+      serverId: gtid.serverId,
+      seqNo: gtid.seqNo,
+    });
+  }
+
+  isEmpty() {
+    return this._domains.size === 0;
+  }
+
+  toString() {
+    return [...this._domains.values()]
+      .sort((a, b) => a.domainId - b.domainId)
+      .map(({ domainId, serverId, seqNo }) =>
+        `${domainId}-${serverId}-${seqNo}`)
+      .join(',');
+  }
+}
+
+function parseUInt32(text, context) {
+  // Decimal digits only: Number() alone would also accept '1e3' or '0x10'
+  if (!/^\d+$/.test(text)) {
+    throw new Error(`Invalid MariaDB GTID in '${context}': '${text}'`);
+  }
+  const value = Number(text);
+  if (!Number.isSafeInteger(value) || value > 0xffffffff) {
+    throw new Error(`Invalid MariaDB GTID in '${context}': '${text}'`);
+  }
+  return value;
+}
+
+function parseSeqNo(text, context) {
+  if (!/^\d+$/.test(text)) {
+    throw new Error(`Invalid MariaDB GTID in '${context}': '${text}'`);
+  }
+  // seq_no is u64; keep the Number-or-exact-string convention used for
+  // other 64-bit values
+  const value = Number(text);
+  if (Number.isSafeInteger(value)) {
+    return value;
+  }
+  if (BigInt(text) > 0xffffffffffffffffn) {
+    throw new Error(`Invalid MariaDB GTID in '${context}': '${text}'`);
+  }
+  return text;
+}
+
+export { MariadbGtidPosition };

package/lib/packet/binlog.js

@@ -17,7 +17,7 @@ export default function initBinlogPacketClass(zongji) {
 
       const timestamp = parser.parseUnsignedNumber(4) * 1000;
       const eventType = parser.parseUnsignedNumber(1);
-      parser.parseUnsignedNumber(4); // serverId
+      const serverId = parser.parseUnsignedNumber(4);
       const eventLength = parser.parseUnsignedNumber(4);
       const nextPosition = parser.parseUnsignedNumber(4);
       parser.parseUnsignedNumber(2); // flags
@@ -27,7 +27,13 @@ export default function initBinlogPacketClass(zongji) {
         nextPosition: nextPosition,
         size: eventLength - BinlogPacket.Length,
         eventType: eventType,
+        // MariaDB GTID events need it: the GTID's server component lives
+        // in the common header
+        serverId: serverId,
       };
+      // Exposed for bookkeeping on events that are filtered out before
+      // being parsed (the event object never exists for those)
+      this.nextPosition = nextPosition;
 
       const EventClass = getEventClass(eventType);
       this.eventName = EventClass.name;

package/lib/rows_event.js

@@ -1,10 +1,26 @@
 import { BinlogEvent } from './binlog_event.js';
 import * as Common from './common.js';
+import { Parser } from './reader.js';
 
 const Version2Events = [
   0x1e, // WRITE_ROWS_EVENT_V2,
   0x1f, // UPDATE_ROWS_EVENT_V2,
   0x20, // DELETE_ROWS_EVENT_V2
+  0xa9, // WRITE_ROWS_COMPRESSED_EVENT (MariaDB, v2 layout)
+  0xaa, // UPDATE_ROWS_COMPRESSED_EVENT
+  0xab, // DELETE_ROWS_COMPRESSED_EVENT
+];
+
+// MariaDB log_bin_compress=ON variants: identical to the corresponding
+// plain event through the columns-present bitmap(s); the row-image area
+// (from the first row's null bitmap) is a compressed-payload envelope
+const CompressedEvents = [
+  0xa6, // WRITE_ROWS_COMPRESSED_EVENT_V1
+  0xa7, // UPDATE_ROWS_COMPRESSED_EVENT_V1
+  0xa8, // DELETE_ROWS_COMPRESSED_EVENT_V1
+  0xa9, // WRITE_ROWS_COMPRESSED_EVENT
+  0xaa, // UPDATE_ROWS_COMPRESSED_EVENT
+  0xab, // DELETE_ROWS_COMPRESSED_EVENT
 ];
 
 const CHECKSUM_SIZE = 4;
@@ -72,20 +88,36 @@ class RowsEvent extends BinlogEvent {
         this.columns_present_bitmap2 = parser.parseBuffer(columnsPresentBitmapSize);
       }
 
-      if (this.useChecksum) {
+      let rowsParser = parser;
+      let trimChecksum = this.useChecksum;
+      if (CompressedEvents.indexOf(options.eventType) !== -1) {
+        // Inflate the row-image area and read the rows from the inflated
+        // bytes; the event's CRC covers the compressed bytes and stays on
+        // the original parser
+        const end =
+          parser._packetEnd - (this.useChecksum ? CHECKSUM_SIZE : 0);
+        const inflated = Common.uncompressBinlogEventPayload(
+          parser.parseBuffer(end - parser._offset));
+        parser._offset = parser._packetEnd;
+        rowsParser = new Parser();
+        rowsParser.append(inflated);
+        trimChecksum = false;
+      }
+
+      if (trimChecksum) {
         // Ignore the checksum at the end of this packet
-        parser._packetEnd -= CHECKSUM_SIZE;
+        rowsParser._packetEnd -= CHECKSUM_SIZE;
       }
 
       this.rows = [];
-      while (!parser.reachedPacketEnd()) {
-        this.rows.push(this._fetchOneRow(parser));
+      while (!rowsParser.reachedPacketEnd()) {
+        this.rows.push(this._fetchOneRow(rowsParser));
       }
 
-      if (this.useChecksum) {
+      if (trimChecksum) {
         // Skip past the checksum at the end of the packet
-        parser._packetEnd += CHECKSUM_SIZE;
-        parser._offset += CHECKSUM_SIZE;
+        rowsParser._packetEnd += CHECKSUM_SIZE;
+        rowsParser._offset += CHECKSUM_SIZE;
       }
     }
   }