@vlasky/zongji 0.6.1 → 0.7.1

package/index.js

@@ -7,23 +7,18 @@ const ConnectionConfigMap = {
   'Pool': obj => obj.config.connectionConfig,
 };
 
-const TableInfoQueryTemplate = `SELECT
+const TableInfoQuery = `SELECT
   COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME,
   COLUMN_COMMENT, COLUMN_TYPE
-  FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'
+  FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=?
   ORDER BY ORDINAL_POSITION`;
 
-// Simple format function to replace %s placeholders
-function formatSql(template, ...args) {
-  let i = 0;
-  return template.replace(/%s/g, () => args[i++]);
-}
-
 class ZongJi extends EventEmitter {
   constructor(dsn) {
     super();
 
     this._pendingErrors = [];
+    this._pendingErrorTimer = null;
     this.on('newListener', (event) => {
       if (event !== 'error' || this._pendingErrors.length === 0) {
         return;
@@ -39,8 +34,12 @@ class ZongJi extends EventEmitter {
     this._filters({});
     this.ctrlCallbacks = [];
     this.tableMap = {};
+    this._warnedUnsupported = new Set();
+    this._currentGtid = undefined;
     this.ready = false;
     this.stopped = false;
+    this._starting = false;
+    this._startEpoch = 0;
     this.useChecksum = false;
 
     this._dsn = dsn;
@@ -54,7 +53,12 @@ class ZongJi extends EventEmitter {
     const createConnection = (options) => {
       const emitError = (err) => {
         if (this.listenerCount('error') === 0) {
+          // Buffer errors that fire before the caller attaches a listener
+          // (typically within the same tick as construction/start). If no
+          // listener ever appears, fall back to EventEmitter's default
+          // behaviour (throw) rather than swallowing failures silently.
           this._pendingErrors.push(err);
+          this._schedulePendingErrorCheck();
           return;
         }
         this.emit('error', err);
@@ -95,6 +99,23 @@ class ZongJi extends EventEmitter {
     this.connection = createConnection(binlogDsn);
   }
 
+  _schedulePendingErrorCheck() {
+    if (this._pendingErrorTimer) {
+      return;
+    }
+    this._pendingErrorTimer = setImmediate(() => {
+      this._pendingErrorTimer = null;
+      if (this.listenerCount('error') > 0 || this._pendingErrors.length === 0) {
+        return;
+      }
+      const pending = this._pendingErrors.slice();
+      this._pendingErrors.length = 0;
+      // No 'error' listener was attached within a macrotask; re-emit so
+      // Node's unhandled 'error' semantics apply (throws)
+      pending.forEach(err => this.emit('error', err));
+    });
+  }
+
   _isChecksumEnabled(next) {
     const SelectChecksumParamSql = 'select @@GLOBAL.binlog_checksum as checksum';
     const SetChecksumSql = 'set @master_binlog_checksum=@@global.binlog_checksum';
@@ -184,19 +205,28 @@ class ZongJi extends EventEmitter {
   }
 
   _fetchTableInfo(tableMapEvent, next) {
-    const sql = formatSql(TableInfoQueryTemplate,
-      tableMapEvent.schemaName, tableMapEvent.tableName);
-
     if (!this.ctrlConnection ||
         this.ctrlConnection.state === 'disconnected' ||
         this.ctrlConnection._fatalError ||
         this.ctrlConnection._protocolError ||
         this.ctrlConnection._closing) {
+      // The binlog connection stays paused, so processing has halted.
+      // During stop() that is expected; otherwise it must not be silent.
+      if (!this.stopped) {
+        this.emit('error', new Error(
+          'Control connection unavailable while fetching column metadata ' +
+          'for ' + tableMapEvent.schemaName + '.' + tableMapEvent.tableName +
+          '. Binlog processing has halted; call stop() then start() to resume.'));
+      }
       return;
     }
 
+    const params = [tableMapEvent.schemaName, tableMapEvent.tableName];
     try {
-      this.ctrlConnection.query(sql, (err, rows) => {
+      // execute() uses a server-side prepared statement: parameters are sent
+      // out-of-band (never spliced into SQL text) and the statement is cached
+      // per connection, so repeated metadata lookups avoid re-parsing.
+      this.ctrlConnection.execute(TableInfoQuery, params, (err, rows) => {
       if (err) {
         // Errors should be emitted
         this.emit('error', err);
@@ -234,12 +264,14 @@ class ZongJi extends EventEmitter {
     filename,
     position,
     startAtEnd,
+    nonBlock,
   } = {}) {
     this.options = {
       serverId,
       filename,
       position,
       startAtEnd,
+      nonBlock,
     };
   }
 
@@ -257,6 +289,28 @@ class ZongJi extends EventEmitter {
       includeSchema,
       excludeSchema,
     };
+
+    // Precompiled lookups so per-event filtering is O(1). Only own keys of
+    // the schema objects are considered (no prototype leakage).
+    const compileSchemaFilter = (schema) => {
+      if (schema === undefined || schema === null) {
+        return undefined;
+      }
+      const compiled = new Map();
+      for (const database of Object.keys(schema)) {
+        const tables = schema[database];
+        compiled.set(database, tables === true ?
+          true : new Set(Array.isArray(tables) ? tables : []));
+      }
+      return compiled;
+    };
+
+    this._includeEventsSet = includeEvents === undefined ?
+      undefined : new Set(Array.isArray(includeEvents) ? includeEvents : []);
+    this._excludeEventsSet =
+      new Set(Array.isArray(excludeEvents) ? excludeEvents : []);
+    this._includeSchemaMap = compileSchemaFilter(includeSchema);
+    this._excludeSchemaMap = compileSchemaFilter(excludeSchema) || new Map();
   }
 
   get(name) {
@@ -289,6 +343,27 @@ class ZongJi extends EventEmitter {
       return;
     }
 
+    // A duplicate start() while one is already initialising is ignored -
+    // the first call completes - but its filters are applied, exactly as
+    // in the already-running branch above: filters are snapshotted, and
+    // re-calling start() is the documented way to update them, so updates
+    // made during the initialisation window must not be lost. Stream
+    // options (filename/position/serverId) still come from the first
+    // call. If stop() intervened, this start() instead proceeds as a
+    // restart: the epoch below makes the stale initialisation chain
+    // abort, so exactly one binlog dump command is ever enqueued.
+    if (this._starting && !this.stopped) {
+      this._filters(options);
+      return;
+    }
+    this._starting = true;
+    this._startEpoch += 1;
+    const epoch = this._startEpoch;
+
+    // A resumed stream must not attribute early events to a GTID seen
+    // before the restart
+    this._currentGtid = undefined;
+
     this.stopped = false;
 
     if (!this.connection || !this.ctrlConnection) {
@@ -330,6 +405,16 @@ class ZongJi extends EventEmitter {
       });
     };
 
+    // Attach the current transaction's GTID (tracked at the packet layer,
+    // even when 'gtid' events are filtered out). Gtid/AnonymousGtid events
+    // keep their own parsed value.
+    const attachGtid = (event) => {
+      if (!('gtid' in event)) {
+        event.gtid = this._currentGtid;
+      }
+      return event;
+    };
+
     const binlogHandler = (error, event) => {
       if (error) {
         return this.emit('error', error);
@@ -347,9 +432,20 @@ class ZongJi extends EventEmitter {
           if (!tableMap || tableMap.tableName !== event.tableName || tableMap.columns.length !== event.columnCount) {
             if (!this.connection) return;
             this.connection.pause();
+            attachGtid(event);
             this._fetchTableInfo(event, () => {
-              // merge the column info with metadata
-              event.updateColumnInfo();
+              try {
+                // merge the column info with metadata
+                event.updateColumnInfo();
+              } catch (err) {
+                // Schema drift between binlog write and metadata fetch:
+                // drop the cached entry (subsequent row events for this
+                // table id are skipped rather than misdecoded) and report
+                delete this.tableMap[event.tableId];
+                this.emit('error', err);
+                if (this.connection) this.connection.resume();
+                return;
+              }
               this.emit('binlog', event);
               if (this.connection) this.connection.resume();
             });
@@ -358,13 +454,32 @@ class ZongJi extends EventEmitter {
           break;
         }
         case 'Rotate':
-          if (this.options.filename !== event.binlogName) {
-            this.options.filename = event.binlogName;
-          }
+          // The payload position is the first event of the NEW file: the
+          // only value coherent with binlogName. The header nextPosition
+          // refers to the OLD file (0 for the artificial rotate at dump
+          // start), so rotates are excluded from the generic update below;
+          // persisting (new filename, old-file offset) would corrupt the
+          // resume point until the next non-rotate event repaired it.
+          this.options.filename = event.binlogName;
+          this.options.position = event.position;
           break;
       }
-      this.options.position = event.nextPosition;
-      this.emit('binlog', event);
+      // Never advance the resume position past a TableMap: a consumer
+      // persisting options.position could otherwise resume in the gap
+      // between a TableMap and its row events, and with no cached table
+      // metadata those rows would be silently dropped. Holding position
+      // back means the TableMap is replayed before its rows on resume;
+      // rows already seen may be re-emitted (at-least-once), which is
+      // recoverable where dropping is not. A narrower window remains for
+      // multi-table statements (all TableMaps precede all row events, so
+      // emitting the first table's rows advances past the later
+      // TableMaps); the complete fix is advancing only at transaction
+      // boundaries, tracked pre-filter.
+      const typeName = event.getTypeName();
+      if (typeName !== 'TableMap' && typeName !== 'Rotate') {
+        this.options.position = event.nextPosition;
+      }
+      this.emit('binlog', attachGtid(event));
     };
 
     let promises = [new Promise(testChecksum)];
@@ -375,7 +490,13 @@ class ZongJi extends EventEmitter {
 
     Promise.all(promises)
       .then(() => {
-        // Check if stop() was called while promises were pending
+        // Abort if a newer start() superseded this one (after a stop())
+        // while promises were pending
+        if (epoch !== this._startEpoch) {
+          return;
+        }
+        this._starting = false;
+        // Abort if stop() was called while promises were pending
         if (this.stopped) {
           return;
         }
@@ -384,8 +505,9 @@ class ZongJi extends EventEmitter {
         this.ready = true;
         this.emit('ready');
 
-        // Final check right before addCommand - connection may have been destroyed
-        if (this.stopped) {
+        // Final check right before addCommand - connection may have been
+        // destroyed or this start() superseded by a ready handler
+        if (this.stopped || epoch !== this._startEpoch) {
           return;
         }
 
@@ -413,6 +535,10 @@ class ZongJi extends EventEmitter {
         this.connection.addCommand(new this.BinlogClass(binlogHandler));
       })
       .catch(err => {
+        if (epoch !== this._startEpoch) {
+          return;
+        }
+        this._starting = false;
         this.emit('error', err);
       });
 
@@ -427,10 +553,25 @@ class ZongJi extends EventEmitter {
       return;
     }
 
+    // Errors emitted by a connection we are deliberately destroying are
+    // teardown noise (e.g. in-flight queries failing with
+    // ERR_STREAM_WRITE_AFTER_END); do not forward them to the caller
+    const silenceErrors = (conn) => {
+      conn.removeAllListeners('error');
+      conn.on('error', () => {});
+    };
+
+    // Capture the connections this stop() owns: finish() may fire from an
+    // async callback after a subsequent start() has already created
+    // replacement connections, and must not touch those
+    const ctrlConnection = this.ctrlConnection;
+    const ctrlToClose = this.ctrlConnectionOwner ? ctrlConnection : null;
+
     // Binary log connection does not end with destroy()
     let connectionThreadId = null;
     if (this.connection) {
       connectionThreadId = this.connection.threadId;
+      silenceErrors(this.connection);
       this.connection.destroy();
       // @ts-ignore - internal mysql2 API
       if (this.connection.stream && typeof this.connection.stream.unref === 'function') {
@@ -443,24 +584,29 @@ class ZongJi extends EventEmitter {
     const finish = () => {
       if (finished) return;
       finished = true;
-      if (this.ctrlConnectionOwner && this.ctrlConnection) {
-        this.ctrlConnection.destroy();
+      if (ctrlToClose) {
+        silenceErrors(ctrlToClose);
+        ctrlToClose.destroy();
         // @ts-ignore - internal mysql2 API
-        if (this.ctrlConnection.stream && typeof this.ctrlConnection.stream.unref === 'function') {
+        if (ctrlToClose.stream && typeof ctrlToClose.stream.unref === 'function') {
           // @ts-ignore - internal mysql2 API
-          this.ctrlConnection.stream.unref();
+          ctrlToClose.stream.unref();
+        }
+        if (this.ctrlConnection === ctrlToClose) {
+          this.ctrlConnection = null;
         }
-        this.ctrlConnection = null;
       }
       this.emit('stopped');
     };
 
-    if (!this.ctrlConnection ||
-        this.ctrlConnection.state === 'disconnected' ||
-        this.ctrlConnection._fatalError ||
-        this.ctrlConnection._protocolError ||
-        this.ctrlConnection._closing) {
-      this.ctrlConnection = null;
+    if (!ctrlConnection ||
+        ctrlConnection.state === 'disconnected' ||
+        ctrlConnection._fatalError ||
+        ctrlConnection._protocolError ||
+        ctrlConnection._closing) {
+      if (this.ctrlConnection === ctrlConnection) {
+        this.ctrlConnection = null;
+      }
       return finish();
     }
 
@@ -470,7 +616,7 @@ class ZongJi extends EventEmitter {
 
     const killTimeout = setTimeout(finish, 1000);
     try {
-      this.ctrlConnection.query(
+      ctrlConnection.query(
         'KILL ' + connectionThreadId,
         () => {
           clearTimeout(killTimeout);
@@ -483,44 +629,40 @@ class ZongJi extends EventEmitter {
     }
   }
 
+  // Emit an error the first time an undecodable event type arrives so that
+  // dropped row changes (e.g. compressed transactions) are never silent.
+  _warnUnsupportedEvent(EventClass) {
+    if (this._warnedUnsupported.has(EventClass.name)) {
+      return;
+    }
+    this._warnedUnsupported.add(EventClass.name);
+    this.emit('error', new Error(EventClass.unsupportedReason));
+  }
+
   // It includes every events by default.
   _skipEvent(name) {
-    const includes = this.filters.includeEvents;
-    const excludes = this.filters.excludeEvents;
-
-    let included = (includes === undefined) ||
-      (Array.isArray(includes) && (includes.indexOf(name) > -1));
-    let excluded = Array.isArray(excludes) && (excludes.indexOf(name) > -1);
-
-    return excluded || !included;
+    if (this._excludeEventsSet.has(name)) {
+      return true;
+    }
+    return this._includeEventsSet !== undefined &&
+      !this._includeEventsSet.has(name);
   }
 
   // It doesn't skip any schema by default.
   _skipSchema(database, table) {
-    const includes = this.filters.includeSchema;
-    const excludes = this.filters.excludeSchema || {};
-
-    let included = (includes === undefined) ||
-      (
-        (database in includes) &&
-        (
-          includes[database] === true ||
-          (
-            Array.isArray(includes[database]) &&
-            includes[database].indexOf(table) > -1
-          )
-        )
-      );
-    let excluded = (database in excludes) &&
-      (
-        excludes[database] === true ||
-        (
-          Array.isArray(excludes[database]) &&
-          excludes[database].indexOf(table) > -1
-        )
-      );
+    const excludeEntry = this._excludeSchemaMap.get(database);
+    if (excludeEntry === true ||
+        (excludeEntry instanceof Set && excludeEntry.has(table))) {
+      return true;
+    }
 
-    return excluded || !included;
+    if (this._includeSchemaMap === undefined) {
+      return false;
+    }
+    const includeEntry = this._includeSchemaMap.get(database);
+    const included = includeEntry === true ||
+      (includeEntry instanceof Set && includeEntry.has(table));
+    return !included;
   }
 }
 

package/lib/binlog_event.js

@@ -261,6 +261,16 @@ class TableMap extends BinlogEvent {
     const tableMap = this.tableMap[this.tableId];
 
     const columnSchemas = tableMap.columnSchemas;
+    // Schema drift: the table was altered between this binlog event being
+    // written and the metadata fetch. Fail diagnosably rather than with a
+    // bare TypeError from indexing missing columns.
+    if (!columnSchemas || columnSchemas.length < this.columnCount) {
+      throw new Error(
+        `Table ${this.schemaName}.${this.tableName} schema changed between ` +
+        `binlog event and metadata fetch: the event has ${this.columnCount} ` +
+        'columns, fetched metadata has ' +
+        `${columnSchemas ? columnSchemas.length : 0}`);
+    }
     const columns = [];
     for (let j = 0; j < this.columnCount; j++) {
       columns.push({
@@ -364,6 +374,43 @@ class Unknown extends BinlogEvent {
   }
 }
 
+/* MySQL 8.0.20+ wraps whole transactions in a single compressed event when
+ * binlog_transaction_compression=ON. The embedded row events are
+ * zstd-compressed and zongji cannot decode them, so the row changes they
+ * carry would be silently lost. The first occurrence emits an error via
+ * ZongJi#_warnUnsupportedEvent (see unsupportedReason below).
+ */
+class TransactionPayload extends BinlogEvent {
+  constructor(parser, options, zongji) {
+    super(parser, options);
+    skipEventRemainder(parser, zongji);
+  }
+
+  static unsupportedReason =
+    'Server sent a TRANSACTION_PAYLOAD_EVENT (binlog_transaction_compression=ON). ' +
+    'zongji cannot decode compressed transactions, so their row events are NOT ' +
+    'being emitted. Set binlog_transaction_compression=OFF on the server to ' +
+    'receive these changes.';
+}
+
+/* MySQL 8.0+ emits partial JSON row updates when
+ * binlog_row_value_options=PARTIAL_JSON. The JSON diff format is not
+ * decodable by zongji, so these updates would be silently lost. The first
+ * occurrence emits an error via ZongJi#_warnUnsupportedEvent.
+ */
+class PartialUpdateRows extends BinlogEvent {
+  constructor(parser, options, zongji) {
+    super(parser, options);
+    skipEventRemainder(parser, zongji);
+  }
+
+  static unsupportedReason =
+    'Server sent a PARTIAL_UPDATE_ROWS_EVENT (binlog_row_value_options=PARTIAL_JSON). ' +
+    'zongji cannot decode partial JSON updates, so these UPDATE row events are ' +
+    'NOT being emitted. Clear binlog_row_value_options on the server to receive ' +
+    'these changes.';
+}
+
 export {
   BinlogEvent,
   Rotate,
@@ -375,5 +422,7 @@ export {
   Gtid,
   AnonymousGtid,
   PreviousGtids,
+  TransactionPayload,
+  PartialUpdateRows,
   Unknown
 };

package/lib/code_map.js

@@ -37,7 +37,13 @@ const CodeEvent = [
   'DELETE_ROWS_EVENT_V2',
   'GTID_LOG_EVENT',
   'ANONYMOUS_GTID_LOG_EVENT',
-  'PREVIOUS_GTIDS_LOG_EVENT'
+  'PREVIOUS_GTIDS_LOG_EVENT',
+  'TRANSACTION_CONTEXT_EVENT',
+  'VIEW_CHANGE_EVENT',
+  'XA_PREPARE_LOG_EVENT',
+  'PARTIAL_UPDATE_ROWS_EVENT',
+  'TRANSACTION_PAYLOAD_EVENT',
+  'HEARTBEAT_LOG_EVENT_V2'
 ];
 
 const EventClass = {
@@ -50,6 +56,8 @@ const EventClass = {
   GTID_LOG_EVENT: events.Gtid,
   ANONYMOUS_GTID_LOG_EVENT: events.AnonymousGtid,
   PREVIOUS_GTIDS_LOG_EVENT: events.PreviousGtids,
+  TRANSACTION_PAYLOAD_EVENT: events.TransactionPayload,
+  PARTIAL_UPDATE_ROWS_EVENT: events.PartialUpdateRows,
 
   TABLE_MAP_EVENT: events.TableMap,
   DELETE_ROWS_EVENT_V1: rowsEvents.DeleteRows,