@powersync/service-module-mysql 0.6.5 → 0.7.0

package/dist/replication/zongji/BinLogListener.js

@@ -0,0 +1,192 @@
+import * as common from '../../common/common-index.js';
+import async from 'async';
+import * as zongji_utils from './zongji-utils.js';
+import { logger as defaultLogger } from '@powersync/lib-services-framework';
+// Maximum time the processing queue can be paused before resuming automatically
+// MySQL server will automatically terminate replication connections after 60 seconds of inactivity, so this guards against that.
+const MAX_QUEUE_PAUSE_TIME_MS = 45_000;
+/**
+ *  Wrapper class for the Zongji BinLog listener. Internally handles the creation and management of the listener and posts
+ *  events on the provided BinLogEventHandler.
+ */
+export class BinLogListener {
+    options;
+    connectionManager;
+    eventHandler;
+    binLogPosition;
+    currentGTID;
+    logger;
+    zongji;
+    processingQueue;
+    /**
+     *  The combined size in bytes of all the binlog events currently in the processing queue.
+     */
+    queueMemoryUsage;
+    constructor(options) {
+        this.options = options;
+        this.logger = options.logger ?? defaultLogger;
+        this.connectionManager = options.connectionManager;
+        this.eventHandler = options.eventHandler;
+        this.binLogPosition = options.startPosition;
+        this.currentGTID = null;
+        this.processingQueue = async.queue(this.createQueueWorker(), 1);
+        this.queueMemoryUsage = 0;
+        this.zongji = this.createZongjiListener();
+    }
+    /**
+     *  The queue memory limit in bytes as defined in the connection options.
+     *  @private
+     */
+    get queueMemoryLimit() {
+        return this.connectionManager.options.binlog_queue_memory_limit * 1024 * 1024;
+    }
+    async start() {
+        if (this.isStopped) {
+            return;
+        }
+        this.logger.info(`Starting replication. Created replica client with serverId:${this.options.serverId}`);
+        this.zongji.start({
+            // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive
+            // tablemap events always need to be included for the other row events to work
+            includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'],
+            includeSchema: { [this.connectionManager.databaseName]: this.options.includedTables },
+            filename: this.binLogPosition.filename,
+            position: this.binLogPosition.offset,
+            serverId: this.options.serverId
+        });
+        return new Promise((resolve, reject) => {
+            // Handle an edge case where the listener has already been stopped before completing startup
+            if (this.isStopped) {
+                this.logger.info('BinLog listener was stopped before startup completed.');
+                resolve();
+            }
+            this.zongji.on('error', (error) => {
+                if (!this.isStopped) {
+                    this.logger.error('Binlog listener error:', error);
+                    this.stop();
+                    reject(error);
+                }
+                else {
+                    this.logger.warn('Binlog listener error during shutdown:', error);
+                }
+            });
+            this.processingQueue.error((error) => {
+                if (!this.isStopped) {
+                    this.logger.error('BinlogEvent processing error:', error);
+                    this.stop();
+                    reject(error);
+                }
+                else {
+                    this.logger.warn('BinlogEvent processing error during shutdown:', error);
+                }
+            });
+            this.zongji.on('stopped', () => {
+                resolve();
+                this.logger.info('BinLog listener stopped. Replication ended.');
+            });
+        });
+    }
+    stop() {
+        if (!this.isStopped) {
+            this.zongji.stop();
+            this.processingQueue.kill();
+        }
+    }
+    get isStopped() {
+        return this.zongji.stopped;
+    }
+    createZongjiListener() {
+        const zongji = this.connectionManager.createBinlogListener();
+        zongji.on('binlog', async (evt) => {
+            this.logger.info(`Received Binlog event:${evt.getEventName()}`);
+            this.processingQueue.push(evt);
+            this.queueMemoryUsage += evt.size;
+            // When the processing queue grows past the threshold, we pause the binlog listener
+            if (this.isQueueOverCapacity()) {
+                this.logger.info(`Binlog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing Binlog listener.`);
+                zongji.pause();
+                const resumeTimeoutPromise = new Promise((resolve) => {
+                    setTimeout(() => resolve('timeout'), MAX_QUEUE_PAUSE_TIME_MS);
+                });
+                await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]);
+                this.logger.info(`Binlog processing queue backlog cleared. Resuming Binlog listener.`);
+                zongji.resume();
+            }
+        });
+        zongji.on('ready', async () => {
+            // Set a heartbeat interval for the Zongji replication connection
+            // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown
+            // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket.
+            await new Promise((resolve, reject) => {
+                this.zongji.connection.query(
+                // In nanoseconds, 10^9 = 1s
+                'set @master_heartbeat_period=28*1000000000', (error, results, fields) => {
+                    if (error) {
+                        reject(error);
+                    }
+                    else {
+                        this.logger.info('Successfully set up replication connection heartbeat...');
+                        resolve(results);
+                    }
+                });
+            });
+            // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat.
+            // The timeout here must be greater than the master_heartbeat_period.
+            const socket = this.zongji.connection._socket;
+            socket.setTimeout(60_000, () => {
+                this.logger.info('Destroying socket due to replication connection timeout.');
+                socket.destroy(new Error('Replication connection timeout.'));
+            });
+            this.logger.info(`BinLog listener setup complete. Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`);
+        });
+        return zongji;
+    }
+    createQueueWorker() {
+        return async (evt) => {
+            switch (true) {
+                case zongji_utils.eventIsGTIDLog(evt):
+                    this.currentGTID = common.ReplicatedGTID.fromBinLogEvent({
+                        raw_gtid: {
+                            server_id: evt.serverId,
+                            transaction_range: evt.transactionRange
+                        },
+                        position: {
+                            filename: this.binLogPosition.filename,
+                            offset: evt.nextPosition
+                        }
+                    });
+                    await this.eventHandler.onTransactionStart({ timestamp: new Date(evt.timestamp) });
+                    break;
+                case zongji_utils.eventIsRotation(evt):
+                    this.binLogPosition.filename = evt.binlogName;
+                    this.binLogPosition.offset = evt.position;
+                    await this.eventHandler.onRotate();
+                    break;
+                case zongji_utils.eventIsWriteMutation(evt):
+                    await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]);
+                    break;
+                case zongji_utils.eventIsUpdateMutation(evt):
+                    await this.eventHandler.onUpdate(evt.rows.map((row) => row.after), evt.rows.map((row) => row.before), evt.tableMap[evt.tableId]);
+                    break;
+                case zongji_utils.eventIsDeleteMutation(evt):
+                    await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]);
+                    break;
+                case zongji_utils.eventIsXid(evt):
+                    const LSN = new common.ReplicatedGTID({
+                        raw_gtid: this.currentGTID.raw,
+                        position: {
+                            filename: this.binLogPosition.filename,
+                            offset: evt.nextPosition
+                        }
+                    }).comparable;
+                    await this.eventHandler.onCommit(LSN);
+                    break;
+            }
+            this.queueMemoryUsage -= evt.size;
+        };
+    }
+    isQueueOverCapacity() {
+        return this.queueMemoryUsage >= this.queueMemoryLimit;
+    }
+}
+//# sourceMappingURL=BinLogListener.js.map

package/dist/replication/zongji/BinLogListener.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BinLogListener.js","sourceRoot":"","sources":["../../../src/replication/zongji/BinLogListener.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,KAAK,YAAY,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAGpF,gFAAgF;AAChF,iIAAiI;AACjI,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAsBvC;;;GAGG;AACH,MAAM,OAAO,cAAc;IAcN;IAbX,iBAAiB,CAAyB;IAC1C,YAAY,CAAqB;IACjC,cAAc,CAAwB;IACtC,WAAW,CAA+B;IAC1C,MAAM,CAAS;IAEvB,MAAM,CAAS;IACf,eAAe,CAAiC;IAChD;;OAEG;IACH,gBAAgB,CAAS;IAEzB,YAAmB,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QAC/C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACH,IAAY,gBAAgB;QAC1B,OAAO,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;IAChF,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YAChB,qHAAqH;YACrH,8EAA8E;YAC9E,aAAa,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,CAAC;YAChG,aAAa,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;YACrF,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;YACtC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;YACpC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;SACT,CAAC,CAAC;QAE1B,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,4FAA4F;YAC5F,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;gBAC1E,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;oBACnD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACnC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;oBAC1D,IAAI,CAAC,IAAI,EAAE,CAAC;oBACZ,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE,KAAK,CAAC,CAAC;gBAC3E,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;gBAC7B,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,IAAI;QACT,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,IAAY,SAAS;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7B,CAAC;IAEO,oBAAoB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,oBAAoB,EAAE,CAAC;QAE7D,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,IAAI,CAAC;YAElC,mFAAmF;YACnF,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4DAA4D,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,yBAAyB,+BAA+B,CACpJ,CAAC;gBACF,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,oBAAoB,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBACnD,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,uBAAuB,CAAC,CAAC;gBAChE,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC;gBAEzE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;gBACvF,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC5B,iEAAiE;YACjE,iGAAiG;YACjG,qGAAqG;YACrG,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACpC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK;gBAC1B,4BAA4B;gBAC5B,4CAA4C,EAC5C,CAAC,KAAU,EAAE,OAAY,EAAE,MAAW,EAAE,EAAE;oBACxC,IAAI,KAAK,EAAE,CAAC;wBACV,MAAM,CAAC,KAAK,CAAC,CAAC;oBAChB,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;wBAC5E,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC,CACF,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,4HAA4H;YAC5H,qEAAqE;YACrE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAQ,CAAC;YAC/C,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAC7E,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,wDAAwD,IAAI,CAAC,cAAc,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CACrH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB;QACvB,OAAO,KAAK,EAAE,GAAgB,EAAE,EAAE;YAChC,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC;oBACnC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,cAAc,CAAC,eAAe,CAAC;wBACvD,QAAQ,EAAE;4BACR,SAAS,EAAE,GAAG,CAAC,QAAQ;4BACvB,iBAAiB,EAAE,GAAG,CAAC,gBAAgB;yBACxC;wBACD,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;4BACtC,MAAM,EAAE,GAAG,CAAC,YAAY;yBACzB;qBACF,CAAC,CAAC;oBACH,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBACnF,MAAM;gBACR,KAAK,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC;oBACpC,IAAI,CAAC,cAAc,CAAC,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC;oBAC9C,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC;oBAC1C,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;oBACnC,MAAM;gBACR,KAAK,YAAY,CAAC,oBAAoB,CAAC,GAAG,CAAC;oBACzC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrE,MAAM;gBACR,KAAK,YAAY,CAAC,qBAAqB,CAAC,GAAG,CAAC;oBAC1C,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAC9B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EACjC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAC1B,CAAC;oBACF,MAAM;gBACR,KAAK,YAAY,CAAC,qBAAqB,CAAC,GAAG,CAAC;oBAC1C,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;oBACtE,MAAM;gBACR,KAAK,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC;oBAC/B,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC;wBACpC,QAAQ,EAAE,IAAI,CAAC,WAAY,CAAC,GAAG;wBAC/B,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;4BACtC,MAAM,EAAE,GAAG,CAAC,YAAY;yBACzB;qBACF,CAAC,CAAC,UAAU,CAAC;oBACd,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM;YACV,CAAC;YAED,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,IAAI,CAAC;QACpC,CAAC,CAAC;IACJ,CAAC;IAED,mBAAmB;QACjB,OAAO,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC;IACxD,CAAC;CACF"}

package/dist/replication/zongji/zongji-utils.d.ts

@@ -1,7 +1,8 @@
-import { BinLogEvent, BinLogGTIDLogEvent, BinLogMutationEvent, BinLogRotationEvent, BinLogUpdateEvent, BinLogXidEvent } from '@powersync/mysql-zongji';
+import { BinLogEvent, BinLogGTIDLogEvent, BinLogRowEvent, BinLogRotationEvent, BinLogTableMapEvent, BinLogRowUpdateEvent, BinLogXidEvent } from '@powersync/mysql-zongji';
 export declare function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent;
+export declare function eventIsTableMap(event: BinLogEvent): event is BinLogTableMapEvent;
 export declare function eventIsXid(event: BinLogEvent): event is BinLogXidEvent;
 export declare function eventIsRotation(event: BinLogEvent): event is BinLogRotationEvent;
-export declare function eventIsWriteMutation(event: BinLogEvent): event is BinLogMutationEvent;
-export declare function eventIsDeleteMutation(event: BinLogEvent): event is BinLogMutationEvent;
-export declare function eventIsUpdateMutation(event: BinLogEvent): event is BinLogUpdateEvent;
+export declare function eventIsWriteMutation(event: BinLogEvent): event is BinLogRowEvent;
+export declare function eventIsDeleteMutation(event: BinLogEvent): event is BinLogRowEvent;
+export declare function eventIsUpdateMutation(event: BinLogEvent): event is BinLogRowUpdateEvent;

package/dist/replication/zongji/zongji-utils.js

@@ -1,6 +1,9 @@
 export function eventIsGTIDLog(event) {
     return event.getEventName() == 'gtidlog';
 }
+export function eventIsTableMap(event) {
+    return event.getEventName() == 'tablemap';
+}
 export function eventIsXid(event) {
     return event.getEventName() == 'xid';
 }

package/dist/replication/zongji/zongji-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"zongji-utils.js","sourceRoot":"","sources":["../../../src/replication/zongji/zongji-utils.ts"],"names":[],"mappings":"AASA,MAAM,UAAU,cAAc,CAAC,KAAkB;IAC/C,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,SAAS,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAkB;IAC3C,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,QAAQ,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,WAAW,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACtD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,YAAY,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACtD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,YAAY,CAAC;AAC9C,CAAC"}
+{"version":3,"file":"zongji-utils.js","sourceRoot":"","sources":["../../../src/replication/zongji/zongji-utils.ts"],"names":[],"mappings":"AAUA,MAAM,UAAU,cAAc,CAAC,KAAkB;IAC/C,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,SAAS,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,UAAU,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAkB;IAC3C,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAkB;IAChD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,QAAQ,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,WAAW,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACtD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,YAAY,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACtD,OAAO,KAAK,CAAC,YAAY,EAAE,IAAI,YAAY,CAAC;AAC9C,CAAC"}

package/dist/types/types.d.ts

@@ -14,6 +14,7 @@ export interface NormalizedMySQLConnectionConfig {
     client_certificate?: string;
     client_private_key?: string;
     lookup?: LookupFunction;
+    binlog_queue_memory_limit: number;
 }
 export declare const MySQLConnectionConfig: t.Intersection<t.Codec<{
     type: string;
@@ -38,6 +39,7 @@ export declare const MySQLConnectionConfig: t.Intersection<t.Codec<{
     client_certificate: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
     client_private_key: t.OptionalCodec<t.Codec<string, string, string, t.CodecProps>>;
     reject_ip_ranges: t.OptionalCodec<t.Codec<string[], string[], string, t.CodecProps>>;
+    binlog_queue_memory_limit: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
 }>>;
 /**
  * Config input specified when starting services

package/dist/types/types.js

@@ -15,7 +15,9 @@ export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.a
     cacert: t.string.optional(),
     client_certificate: t.string.optional(),
     client_private_key: t.string.optional(),
-    reject_ip_ranges: t.array(t.string).optional()
+    reject_ip_ranges: t.array(t.string).optional(),
+    // The combined size of binlog events that can be queued in memory before throttling is applied.
+    binlog_queue_memory_limit: t.number.optional()
 }));
 /**
  * Validate and normalize connection options.
@@ -61,6 +63,8 @@ export function normalizeConnectionConfig(options) {
         username,
         password,
         server_id: options.server_id ?? 1,
+        // Binlog processing queue memory limit before throttling is applied.
+        binlog_queue_memory_limit: options.binlog_queue_memory_limit ?? 50,
         lookup
     };
 }

package/dist/types/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACxG,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,KAAK,MAAM,QAAQ,CAAC;AAEhC,MAAM,CAAC,MAAM,qBAAqB,GAAG,OAAgB,CAAC;AAqBtD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAChF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACxB,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE;IACnD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAEvC,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CACH,CAAC;AAYF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA8B;IACtE,IAAI,GAAwB,CAAC;IAC7B,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,6CAA6C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAC1E,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;IAEtD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAErE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,IAAI,EAAE,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,IAAI,EAAE,CAAC;IAExD,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,EAAE,EAAE,CAAC,CAAC;IAE1G,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAE7B,QAAQ;QACR,IAAI;QACJ,QAAQ;QAER,QAAQ;QACR,QAAQ;QAER,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC;QAEjC,MAAM;KACP,CAAC;AACJ,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACxG,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,KAAK,MAAM,QAAQ,CAAC;AAEhC,MAAM,CAAC,MAAM,qBAAqB,GAAG,OAAgB,CAAC;AAuBtD,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAChF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACxB,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE;IACnD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAEvC,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAC9C,gGAAgG;IAChG,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC/C,CAAC,CACH,CAAC;AAYF;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA8B;IACtE,IAAI,GAAwB,CAAC;IAC7B,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,6CAA6C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAC1E,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;IAEtD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAErE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,IAAI,EAAE,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,IAAI,EAAE,CAAC;IAExD,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,qCAAqC,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,EAAE,EAAE,CAAC,CAAC;IAE1G,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,SAAS;QAC3B,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,SAAS;QAE7B,QAAQ;QACR,IAAI;QACJ,QAAQ;QAER,QAAQ;QACR,QAAQ;QAER,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC;QAEjC,qEAAqE;QACrE,yBAAyB,EAAE,OAAO,CAAC,yBAAyB,IAAI,EAAE;QAElE,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/utils/mysql-utils.js

@@ -30,6 +30,7 @@ export function createPool(config, options) {
     return mysql.createPool({
         host: config.hostname,
         user: config.username,
+        port: config.port,
         password: config.password,
         database: config.database,
         ssl: hasSSLOptions ? sslOptions : undefined,

package/dist/utils/mysql-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,MAAM,QAAQ,CAAC;AAG3B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAUrC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,KAAK,IAAI,KAAK,GAAG,OAAO,GAAI,KAAK,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;YAC1C,OAAO,UAAU,CAAC,KAAK,CAA+B,KAAK,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA6C,EAAE,OAA2B;IACnG,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,GAAG,EAAE,MAAM,CAAC,kBAAkB;QAC9B,IAAI,EAAE,MAAM,CAAC,kBAAkB;KAChC,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,6CAA6C;IAC7C,OAAO,KAAK,CAAC,UAAU,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC3C,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,GAAG,EAAE,0DAA0D;QACzE,WAAW,EAAE,IAAI,EAAE,iCAAiC;QACpD,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAAmC;IACvE,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,MAAM,YAAY,CAAC;QAC3C,UAAU;QACV,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,OAAiB,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,cAAsB;IACtE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,qBAAqB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAErD,OAAO,GAAG,CAAC,cAAe,EAAE,qBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;AAC9F,CAAC"}
+{"version":3,"file":"mysql-utils.js","sourceRoot":"","sources":["../../src/utils/mysql-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,MAAM,QAAQ,CAAC;AAG3B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAUrC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,KAAK,IAAI,KAAK,GAAG,OAAO,GAAI,KAAK,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAC;YAC1C,OAAO,UAAU,CAAC,KAAK,CAA+B,KAAK,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAA6C,EAAE,OAA2B;IACnG,MAAM,UAAU,GAAG;QACjB,EAAE,EAAE,MAAM,CAAC,MAAM;QACjB,GAAG,EAAE,MAAM,CAAC,kBAAkB;QAC9B,IAAI,EAAE,MAAM,CAAC,kBAAkB;KAChC,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,6CAA6C;IAC7C,OAAO,KAAK,CAAC,UAAU,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,QAAQ;QACrB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC3C,iBAAiB,EAAE,IAAI;QACvB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,GAAG,EAAE,0DAA0D;QACzE,WAAW,EAAE,IAAI,EAAE,iCAAiC;QACpD,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAAmC;IACvE,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,MAAM,YAAY,CAAC;QAC3C,UAAU;QACV,KAAK,EAAE,6BAA6B;KACrC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,OAAiB,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,cAAsB;IACtE,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,qBAAqB,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAErD,OAAO,GAAG,CAAC,cAAe,EAAE,qBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;AAC9F,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mysql",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.6.5",
+  "version": "0.7.0",
   "license": "FSL-1.1-Apache-2.0",
   "main": "dist/index.js",
   "type": "module",
@@ -22,25 +22,25 @@
     }
   },
   "dependencies": {
-    "@powersync/mysql-zongji": "^0.1.0",
+    "@powersync/mysql-zongji": "0.2.0",
     "async": "^3.2.4",
     "mysql2": "^3.11.0",
     "semver": "^7.5.4",
     "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^11.1.0",
-    "@powersync/lib-services-framework": "0.6.0",
-    "@powersync/service-core": "1.12.1",
-    "@powersync/service-jsonbig": "0.17.10",
+    "@powersync/lib-services-framework": "0.7.0",
+    "@powersync/service-core": "1.13.0",
     "@powersync/service-sync-rules": "0.27.0",
-    "@powersync/service-types": "0.11.0"
+    "@powersync/service-types": "0.12.0",
+    "@powersync/service-jsonbig": "0.17.10"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/semver": "^7.5.4",
-    "@powersync/service-core-tests": "0.9.5",
-    "@powersync/service-module-mongodb-storage": "0.9.5",
-    "@powersync/service-module-postgres-storage": "0.7.5"
+    "@powersync/service-core-tests": "0.10.0",
+    "@powersync/service-module-mongodb-storage": "0.10.0",
+    "@powersync/service-module-postgres-storage": "0.8.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/MySQLRouteAPIAdapter.ts

@@ -249,7 +249,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
     };
   }
 
-  async getReplicationLag(options: api.ReplicationLagOptions): Promise<number | undefined> {
+  async getReplicationLagBytes(options: api.ReplicationLagOptions): Promise<number | undefined> {
     const { bucketStorage } = options;
     const lastCheckpoint = await bucketStorage.getCheckpoint();
 

package/src/replication/BinLogReplicationJob.ts

@@ -1,4 +1,4 @@
-import { container } from '@powersync/lib-services-framework';
+import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
 import { replication } from '@powersync/service-core';
 import { BinlogConfigurationError, BinLogStream } from './BinLogStream.js';
 import { MySQLConnectionManagerFactory } from './MySQLConnectionManagerFactory.js';
@@ -9,9 +9,11 @@ export interface BinLogReplicationJobOptions extends replication.AbstractReplica
 
 export class BinLogReplicationJob extends replication.AbstractReplicationJob {
   private connectionFactory: MySQLConnectionManagerFactory;
+  private lastStream: BinLogStream | null = null;
 
   constructor(options: BinLogReplicationJobOptions) {
     super(options);
+    this.logger = defaultLogger.child({ prefix: `[powersync_${this.options.storage.group_id}] ` });
     this.connectionFactory = options.connectionFactory;
   }
 
@@ -31,7 +33,7 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
           replication_slot: this.slot_name
         }
       });
-      this.logger.error(`Replication failed on ${this.slot_name}`, e);
+      this.logger.error(`Replication failed`, e);
     } finally {
       this.abortController.abort();
     }
@@ -61,17 +63,19 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
         return;
       }
       const stream = new BinLogStream({
+        logger: this.logger,
         abortSignal: this.abortController.signal,
         storage: this.options.storage,
         metrics: this.options.metrics,
         connections: connectionManager
       });
+      this.lastStream = stream;
       await stream.replicate();
     } catch (e) {
       if (this.abortController.signal.aborted) {
         return;
       }
-      this.logger.error(`Sync rules ${this.id} Replication error`, e);
+      this.logger.error(`Replication error`, e);
       if (e.cause != null) {
         this.logger.error(`cause`, e.cause);
       }
@@ -92,4 +96,8 @@ export class BinLogReplicationJob extends replication.AbstractReplicationJob {
       await connectionManager.end();
     }
   }
+
+  async getReplicationLagMillis(): Promise<number | undefined> {
+    return this.lastStream?.getReplicationLagMillis();
+  }
 }

package/src/replication/BinLogReplicator.ts

@@ -38,4 +38,29 @@ export class BinLogReplicator extends replication.AbstractReplicator<BinLogRepli
   async testConnection() {
     return await MySQLModule.testConnection(this.connectionFactory.connectionConfig);
   }
+
+  async getReplicationLagMillis(): Promise<number | undefined> {
+    const lag = await super.getReplicationLagMillis();
+    if (lag != null) {
+      return lag;
+    }
+
+    // Booting or in an error loop. Check last active replication status.
+    // This includes sync rules in an ERROR state.
+    const content = await this.storage.getActiveSyncRulesContent();
+    if (content == null) {
+      return undefined;
+    }
+    // Measure the lag from the last commit or keepalive timestamp.
+    // This is not 100% accurate since it is the commit time in the storage db rather than
+    // the source db, but it's the best we currently have for mysql.
+    const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0;
+    const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0;
+    const latestTs = Math.max(checkpointTs, keepaliveTs);
+    if (latestTs != 0) {
+      return Date.now() - latestTs;
+    }
+
+    return undefined;
+  }
 }