@vlasky/zongji 0.7.0 → 0.8.0

package/index.js

@@ -1,6 +1,8 @@
 import mysql from 'mysql2';
 import { EventEmitter } from 'events';
 import initBinlogClass from './lib/sequence/binlog.js';
+import { GtidSet } from './lib/gtid_set.js';
+import { MariadbGtidPosition } from './lib/mariadb_gtid.js';
 
 const ConnectionConfigMap = {
   'Connection': obj => obj.config,
@@ -36,11 +38,18 @@ class ZongJi extends EventEmitter {
     this.tableMap = {};
     this._warnedUnsupported = new Set();
     this._currentGtid = undefined;
+    this._executedGtids = null;
+    this._pendingGtid = undefined;
+    this._positionFrozen = false;
+    this._seedGtidsFromStream = false;
+    this._startFromGtids = false;
     this.ready = false;
     this.stopped = false;
     this._starting = false;
     this._startEpoch = 0;
     this.useChecksum = false;
+    this.serverVersion = undefined;
+    this.isMariaDb = false;
 
     this._dsn = dsn;
     this.ctrlConnection = null;
@@ -192,6 +201,110 @@ class ZongJi extends EventEmitter {
       });
   }
 
+  // The transactions this instance knows to be processed: the start()
+  // seed plus every transaction whose commit has been observed. Persist
+  // it and pass to start({ gtidSet }) to resume, including on a different
+  // server in the same replication topology. A MySQL GTID set
+  // ('uuid:1-5,...') or a MariaDB GTID position ('0-1-1234,...')
+  // depending on the connected server. Undefined when no exact seed was
+  // available (see start()).
+  get gtidSet() {
+    return this._executedGtids ? this._executedGtids.toString() : undefined;
+  }
+
+  // The GTID of the transaction whose events are currently being
+  // delivered (what event.gtid is for them); undefined between
+  // transactions. Only coherent when read synchronously inside a
+  // 'binlog' handler: checkpointing code should use event.gtid.
+  get lastGtid() {
+    return this._currentGtid;
+  }
+
+  // Called from the packet layer (before event filtering) for
+  // GTID-relevant events. A transaction's GTID enters the executed set
+  // only once its commit marker has been seen (Xid, or a Query event
+  // other than BEGIN, e.g. DDL or COMMIT), so a persisted zongji.gtidSet
+  // never claims a transaction whose row events were still in flight.
+  // Returns true when the event definitively closes the current
+  // transaction, so the caller can stop attaching its GTID to
+  // subsequent events.
+  _trackGtidProgress(eventName, event) {
+    const fold = () => {
+      if (this._pendingGtid !== undefined && this._executedGtids) {
+        this._executedGtids.add(this._pendingGtid);
+      }
+      this._pendingGtid = undefined;
+    };
+
+    switch (eventName) {
+      case 'gtid':
+        // A new transaction implies the previous one committed, so this
+        // also covers commit markers hidden by event filtering
+        fold();
+        this._pendingGtid = event.gtid;
+        break;
+      case 'anonymousgtid':
+        fold();
+        break;
+      case 'xid':
+        fold();
+        return true;
+      case 'xaprepare':
+        // Ends the group's event delivery on both flavours, but must NOT
+        // fold: the prepared transaction commits later under the XA
+        // COMMIT statement's own GTID, and the pending GTID folds when
+        // that (or any next) GTID event arrives
+        return true;
+      case 'query': {
+        // Fold only on definite commit markers. Anything else (BEGIN,
+        // XA START/END, DDL, SAVEPOINT, ...) must not fold: claiming a
+        // transaction early risks losing its remaining events on resume,
+        // whereas not folding merely delays the checkpoint until the
+        // next transaction's GTID arrives (at-least-once redelivery).
+        const query = event.query.trim().toUpperCase();
+        if (query === 'COMMIT' || query === 'ROLLBACK' ||
+            query.startsWith('XA COMMIT') ||
+            query.startsWith('XA ROLLBACK')) {
+          fold();
+          return true;
+        }
+        break;
+      }
+      case 'previousgtids':
+        // When dumping from the start of a binlog file, its Previous_gtids
+        // event is the exact "everything before this point" seed
+        if (this._seedGtidsFromStream && this._executedGtids === null) {
+          try {
+            this._executedGtids = GtidSet.parse(event.gtidSet);
+          } catch {
+            // Leave unseeded; gtidSet stays undefined
+          }
+        }
+        break;
+      case 'mariadbgtid':
+        // As with 'gtid': a new event group implies the previous one
+        // completed. This also closes standalone groups (DDL), which have
+        // no commit marker at all
+        fold();
+        this._pendingGtid = {
+          domainId: event.domainId,
+          serverId: event.serverId,
+          seqNo: event.seqNo,
+        };
+        break;
+      case 'mariadbgtidlist':
+        // At the start of a binlog file this is the exact "everything
+        // before this point" seed, MariaDB's Previous_gtids analogue.
+        // Fake mid-stream lists on GTID connects never seed: the position
+        // is already non-null then
+        if (this._seedGtidsFromStream && this._executedGtids === null) {
+          this._executedGtids = MariadbGtidPosition.fromGtidList(event.gtids);
+        }
+        break;
+    }
+    return false;
+  }
+
   _findBinlogEnd(next) {
     this.ctrlConnection.query('SHOW BINARY LOGS', (err, rows) => {
       if (err) {
@@ -264,14 +377,18 @@ class ZongJi extends EventEmitter {
     filename,
     position,
     startAtEnd,
+    gtidSet,
     nonBlock,
+    requestAnnotateRows,
   } = {}) {
     this.options = {
       serverId,
       filename,
       position,
       startAtEnd,
+      gtidSet,
       nonBlock,
+      requestAnnotateRows,
     };
   }
 
@@ -343,12 +460,17 @@ class ZongJi extends EventEmitter {
       return;
     }
 
-    // A duplicate start() while one is already initialising is ignored;
-    // the first call completes. But if stop() intervened, this start()
-    // must proceed as a restart: the epoch below makes the stale
-    // initialisation chain abort, so exactly one binlog dump command is
-    // ever enqueued.
+    // A duplicate start() while one is already initialising is ignored -
+    // the first call completes - but its filters are applied, exactly as
+    // in the already-running branch above: filters are snapshotted, and
+    // re-calling start() is the documented way to update them, so updates
+    // made during the initialisation window must not be lost. Stream
+    // options (filename/position/serverId) still come from the first
+    // call. If stop() intervened, this start() instead proceeds as a
+    // restart: the epoch below makes the stale initialisation chain
+    // abort, so exactly one binlog dump command is ever enqueued.
     if (this._starting && !this.stopped) {
+      this._filters(options);
       return;
     }
     this._starting = true;
@@ -356,8 +478,38 @@ class ZongJi extends EventEmitter {
     const epoch = this._startEpoch;
 
     // A resumed stream must not attribute early events to a GTID seen
-    // before the restart
+    // before the restart, nor inherit a position freeze from a
+    // transaction the previous stream was in the middle of
     this._currentGtid = undefined;
+    this._pendingGtid = undefined;
+    this._positionFrozen = false;
+
+    // Executed-GTID-set tracking (drives the zongji.gtidSet checkpoint).
+    // Exact seeds: an explicit start set, the server's gtid_executed for
+    // startAtEnd (fetched below), or the stream's first Previous_gtids
+    // event when dumping from the start of a binlog file. An arbitrary
+    // mid-file file+position start has no exact seed, so gtidSet stays
+    // undefined there.
+    this._executedGtids = null;
+    this._seedGtidsFromStream = false;
+    this._startFromGtids = options.gtidSet != null;
+    if (this._startFromGtids) {
+      // The flavour is sniffed from the format (a MySQL set always
+      // contains ':', a MariaDB position never does; '' is valid for
+      // either) and verified against the server after detection
+      try {
+        this._executedGtids = String(options.gtidSet).includes(':')
+          ? GtidSet.parse(options.gtidSet)
+          : MariadbGtidPosition.parse(options.gtidSet);
+      } catch (err) {
+        this._starting = false;
+        this.emit('error', err);
+        return;
+      }
+    } else if (!options.startAtEnd &&
+        (options.position === undefined || options.position <= 4)) {
+      this._seedGtidsFromStream = true;
+    }
 
     this.stopped = false;
 
@@ -368,6 +520,28 @@ class ZongJi extends EventEmitter {
     this._options(options);
     this._filters(options);
 
+    // Server flavour steers the rest of the preamble: MariaDB has no
+    // gtid_mode/gtid_executed (its GTIDs are domain-server-sequence
+    // watermarks, always on) and needs its own dump-request dialect.
+    // SELECT VERSION() is used rather than the handshake packet because
+    // MariaDB prefixes the handshake version with "5.5.5-" for old-client
+    // compatibility, while VERSION() always reports e.g.
+    // "11.8.8-MariaDB-ubu2404-log".
+    const detectServer = (resolve, reject) => {
+      this.ctrlConnection.query('SELECT VERSION() AS version', (err, rows) => {
+        // Never overwrite state belonging to a newer start()
+        if (epoch !== this._startEpoch) {
+          return resolve();
+        }
+        if (err) {
+          return reject(err);
+        }
+        this.serverVersion = rows[0].version;
+        this.isMariaDb = /mariadb/i.test(this.serverVersion);
+        resolve();
+      });
+    };
+
     const testChecksum = (resolve, reject) => {
       this._isChecksumEnabled((err, checksumEnabled) => {
         if (err) {
@@ -383,6 +557,10 @@ class ZongJi extends EventEmitter {
 
     const findBinlogEnd = (resolve, reject) => {
       this._findBinlogEnd((err, result) => {
+        // As above: never mutate options for a superseded start()
+        if (epoch !== this._startEpoch) {
+          return resolve();
+        }
         if (err) {
           return reject(err);
         }
@@ -400,6 +578,30 @@ class ZongJi extends EventEmitter {
       });
     };
 
+    // For startAtEnd the server's own executed set is the exact seed for
+    // zongji.gtidSet ("everything up to now"); transactions racing between
+    // this query and the dump start are streamed and merge idempotently
+    const seedGtidsFromServer = (resolve, reject) => {
+      this.ctrlConnection.query(
+        'SELECT @@GLOBAL.gtid_executed AS gtidExecuted', (err, rows) => {
+          // A stale seed must not overwrite state belonging to a newer
+          // start() that superseded this one while the query was in flight
+          if (epoch !== this._startEpoch) {
+            return resolve();
+          }
+          if (err) {
+            return reject(err);
+          }
+          try {
+            this._executedGtids =
+              GtidSet.parse(rows[0].gtidExecuted.replace(/\s/g, ''));
+          } catch (parseErr) {
+            return reject(parseErr);
+          }
+          resolve();
+        });
+    };
+
     // Attach the current transaction's GTID (tracked at the packet layer,
     // even when 'gtid' events are filtered out). Gtid/AnonymousGtid events
     // keep their own parsed value.
@@ -423,14 +625,42 @@ class ZongJi extends EventEmitter {
 
       switch (event.getTypeName()) {
         case 'TableMap': {
+          // On MariaDB, classic temporal codes may hide 5.3 "hires"
+          // columns that no binlog metadata (FULL included) can reveal;
+          // such tables must take the INFORMATION_SCHEMA path below
+          if (event.hasSelfDescribingMetadata() &&
+              !(this.isMariaDb && event.hasAmbiguousTemporalColumns())) {
+            // MySQL 8.0+ with binlog_row_metadata=FULL: the event itself
+            // carries complete column metadata as of binlog write time, so
+            // no INFORMATION_SCHEMA round-trip (and no connection pause) is
+            // needed. Rebuilt on every TableMap event, so ALTER TABLE never
+            // leaves stale columns behind.
+            this.tableMap[event.tableId] = {
+              columnSchemas: event.buildColumnSchemas(),
+              parentSchema: event.schemaName,
+              tableName: event.tableName,
+            };
+            event.updateColumnInfo();
+            break;
+          }
           const tableMap = this.tableMap[event.tableId];
           if (!tableMap || tableMap.tableName !== event.tableName || tableMap.columns.length !== event.columnCount) {
             if (!this.connection) return;
             this.connection.pause();
             attachGtid(event);
             this._fetchTableInfo(event, () => {
-              // merge the column info with metadata
-              event.updateColumnInfo();
+              try {
+                // merge the column info with metadata
+                event.updateColumnInfo();
+              } catch (err) {
+                // Schema drift between binlog write and metadata fetch:
+                // drop the cached entry (subsequent row events for this
+                // table id are skipped rather than misdecoded) and report
+                delete this.tableMap[event.tableId];
+                this.emit('error', err);
+                if (this.connection) this.connection.resume();
+                return;
+              }
               this.emit('binlog', event);
               if (this.connection) this.connection.resume();
             });
@@ -438,23 +668,144 @@ class ZongJi extends EventEmitter {
           }
           break;
         }
-        case 'Rotate':
-          if (this.options.filename !== event.binlogName) {
-            this.options.filename = event.binlogName;
-          }
-          break;
       }
-      this.options.position = event.nextPosition;
+      // Rotate events update the (filename, position) resume pair at the
+      // packet layer (lib/sequence/binlog.js), before event filtering, so
+      // the pair stays coherent even when 'rotate' is excluded by
+      // includeEvents. They remain excluded from the generic update below:
+      // the header nextPosition refers to the OLD file (0 for the
+      // artificial rotate at dump start), and persisting (new filename,
+      // old-file offset) would corrupt the resume point.
+      // Never advance the resume position into a transaction's
+      // TableMap-to-commit span: a consumer persisting options.position
+      // could otherwise resume between a TableMap and row events that
+      // depend on it, and with no cached table metadata those rows would
+      // be silently dropped (a multi-table statement writes all its
+      // TableMaps before any row event, so even a position after one
+      // table's rows is inside the danger zone). The packet layer
+      // freezes the position at the first TableMap and unfreezes it at
+      // the commit marker; on resume the whole transaction is replayed,
+      // so rows already seen may be re-emitted (at-least-once), which is
+      // recoverable where dropping is not.
+      // MariaDB writes end_log_pos=0 on every event inside a transaction
+      // (TableMap, row events); only commits and standalone events carry a
+      // real position. Adopting a zero would corrupt the resume point.
+      const typeName = event.getTypeName();
+      if (typeName !== 'TableMap' && typeName !== 'Rotate' &&
+          !this._positionFrozen && event.nextPosition > 0) {
+        this.options.position = event.nextPosition;
+      }
       this.emit('binlog', attachGtid(event));
     };
 
-    let promises = [new Promise(testChecksum)];
+    // Announce MariaDB slave capability 4 ("understands GTID events") on
+    // the binlog connection before the dump command is enqueued. Without
+    // it the server rewrites its own events for pre-GTID clients: GTID
+    // events become literal BEGIN Query events and standalone
+    // GTID/checkpoint events become dummy Query events, so no GTID
+    // information ever reaches the stream.
+    const setMariaDbCapability = (resolve, reject) => {
+      this.connection.query('SET @mariadb_slave_capability=4', (err) => {
+        if (epoch !== this._startEpoch) {
+          return resolve();
+        }
+        if (err) {
+          return reject(err);
+        }
+        resolve();
+      });
+    };
 
-    if (this.options.startAtEnd) {
-      promises.push(new Promise(findBinlogEnd));
-    }
+    // MariaDB has no COM_BINLOG_DUMP_GTID: the resume position travels in
+    // the @slave_connect_state session variable and the dump command is a
+    // plain COM_BINLOG_DUMP whose filename/position the server then
+    // ignores. An empty position means "from the oldest binlog available"
+    const setMariaDbConnectState = (resolve, reject) => {
+      // Validated as digits/dashes/commas by the parser, so safe to inline
+      const position = this._executedGtids.toString();
+      this.connection.query(
+        `SET @slave_connect_state='${position}'`, (err) => {
+          if (epoch !== this._startEpoch) {
+            return resolve();
+          }
+          if (err) {
+            return reject(err);
+          }
+          resolve();
+        });
+    };
+
+    // MariaDB's analogue of the gtid_executed seed below: the last GTID
+    // per domain in the binlog or applied by a slave thread, whichever is
+    // most recent (matches what a real replica would send on connect)
+    const seedMariaDbGtidsFromServer = (resolve, reject) => {
+      this.ctrlConnection.query(
+        'SELECT @@GLOBAL.gtid_current_pos AS gtidPos', (err, rows) => {
+          if (epoch !== this._startEpoch) {
+            return resolve();
+          }
+          if (err) {
+            return reject(err);
+          }
+          try {
+            this._executedGtids =
+              MariadbGtidPosition.parse(rows[0].gtidPos.trim());
+          } catch (parseErr) {
+            return reject(parseErr);
+          }
+          resolve();
+        });
+    };
 
-    Promise.all(promises)
+    // Detection must complete before the rest of the preamble because the
+    // seed query below is MySQL-only
+    new Promise(detectServer)
+      .then(() => {
+        if (this._startFromGtids) {
+          // The start set was parsed by format before the server flavour
+          // was known; an empty position is valid for either flavour,
+          // anything else must match the connected server
+          const isMariaDbPosition =
+            this._executedGtids instanceof MariadbGtidPosition;
+          if (isMariaDbPosition !== this.isMariaDb) {
+            if (this._executedGtids.isEmpty()) {
+              this._executedGtids = this.isMariaDb
+                ? new MariadbGtidPosition()
+                : new GtidSet();
+            } else if (this.isMariaDb) {
+              throw new Error(
+                'gtidSet is a MySQL GTID set but the server is MariaDB; ' +
+                'pass a MariaDB GTID position instead ' +
+                '(domain-server-sequence, e.g. \'0-1-1234\')');
+            } else {
+              throw new Error(
+                'gtidSet is a MariaDB GTID position but the server is ' +
+                'MySQL; pass a MySQL GTID set instead ' +
+                '(e.g. \'uuid:1-5\')');
+            }
+          }
+        }
+
+        let promises = [new Promise(testChecksum)];
+
+        if (this.isMariaDb) {
+          promises.push(new Promise(setMariaDbCapability));
+          if (this._startFromGtids) {
+            promises.push(new Promise(setMariaDbConnectState));
+          }
+        }
+
+        if (this.options.startAtEnd) {
+          promises.push(new Promise(findBinlogEnd));
+          if (this._executedGtids === null) {
+            promises.push(new Promise(this.isMariaDb
+              ? seedMariaDbGtidsFromServer
+              : seedGtidsFromServer));
+          }
+        }
+
+        return Promise.all(promises);
+      })
       .then(() => {
         // Abort if a newer start() superseded this one (after a stop())
         // while promises were pending
@@ -633,3 +984,8 @@ class ZongJi extends EventEmitter {
 }
 
 export default ZongJi;
+// The GTID set model behind zongji.gtidSet, exported so consumers can
+// parse persisted sets and test membership (e.g. "was this event.gtid
+// already covered by my snapshot?") without their own parser. Handles
+// tagged GTIDs (MySQL 8.3+).
+export { GtidSet };