@vlasky/zongji 0.5.9 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/.claude/settings.local.json +16 -0
  2. package/.mcp.json +17 -0
  3. package/.tap/processinfo/84de694d-068b-4ac7-b4d8-f835c2122b2d.json +892 -0
  4. package/.tap/processinfo/a1e9627e-66eb-412c-aaa5-079a2eb9f385.json +891 -0
  5. package/.tap/processinfo/abc34c6f-f4aa-4405-8f9a-947048be1df2.json +892 -0
  6. package/.tap/processinfo/affe6c5b-a34d-41b9-bb73-4b14d35138de.json +891 -0
  7. package/.tap/processinfo/bf82449d-cdeb-478b-9506-b939a9506bf2.json +883 -0
  8. package/.tap/processinfo/dd98c508-7067-4342-bdf5-63664b630701.json +892 -0
  9. package/.tap/test-results/test_await_end.js.tap +19 -0
  10. package/.tap/test-results/test_await_end2.js.tap +27 -0
  11. package/.tap/test-results/test_debug_init.js.tap +23 -0
  12. package/.tap/test-results/test_debug_init2.js.tap +35 -0
  13. package/.tap/test-results/test_invalid_host.js.tap +34 -0
  14. package/.tap/test-results/test_minimal.js.tap +26 -0
  15. package/.tap/test-results/test_queryseq.js.tap +14 -0
  16. package/.tap/test-results/test_timing.js.tap +17 -0
  17. package/.tap/test-results/test_unref.js.tap +15 -0
  18. package/CHANGELOG.md +55 -0
  19. package/CLAUDE.md +99 -0
  20. package/GTID.md +54 -0
  21. package/README.md +44 -21
  22. package/TODO.md +43 -0
  23. package/docker-compose.yml +14 -13
  24. package/docker-test.sh +7 -7
  25. package/eslint.config.js +35 -0
  26. package/example.js +1 -1
  27. package/index.d.ts +314 -0
  28. package/index.js +444 -272
  29. package/jsconfig.json +15 -0
  30. package/lib/binlog_event.js +296 -217
  31. package/lib/code_map.js +7 -4
  32. package/lib/common.js +35 -22
  33. package/lib/datetime_decode.js +147 -34
  34. package/lib/json_decode.js +12 -6
  35. package/lib/packet/binlog.js +3 -3
  36. package/lib/packet/combinlog.js +22 -20
  37. package/lib/packet/index.js +50 -50
  38. package/lib/reader.js +215 -109
  39. package/lib/rows_event.js +98 -102
  40. package/lib/sequence/binlog.js +142 -39
  41. package/notes.md +77 -0
  42. package/package.json +15 -10
  43. package/scripts/start-mysql.sh +28 -0
  44. package/workerthreads.md +173 -0
  45. package/.eslintrc +0 -19
@@ -0,0 +1,19 @@
1
+ Test file starting
2
+ TAP version 14
3
+ Invalid host test starting
4
+ Got error: ENOTFOUND
5
+ # Subtest: Connect to an invalid host
6
+ ok 1 - should be equal
7
+ Invalid host test ending
8
+ 1..1
9
+ ok 1 - Connect to an invalid host # time=112.602ms
10
+
11
+ MySQL test starting
12
+ Query callback! err: null
13
+ # Subtest: Simple MySQL query
14
+ ok 1 - query worked
15
+ MySQL test ending
16
+ 1..1
17
+ ok 2 - Simple MySQL query # time=11.899ms
18
+
19
+ 1..2
@@ -0,0 +1,27 @@
1
+ TAP version 14
2
+ # Subtest: Connect to an invalid host
3
+ ok 1 - should be equal
4
+ 1..1
5
+ ok 1 - Connect to an invalid host # time=11.133ms
6
+
7
+ # Subtest: Simple MySQL query
8
+ not ok 1 - test unfinished
9
+ ---
10
+ at:
11
+ fileName: test_await_end2.js
12
+ lineNumber: 35
13
+ columnNumber: 5
14
+ isToplevel: true
15
+ stack: |
16
+ test_await_end2.js:35:5
17
+ test: Simple MySQL query
18
+ source: |
19
+ });
20
+
21
+ tap.test('Simple MySQL query', async test => {
22
+ ----^
23
+ const conn = mysql.createConnection(settings);
24
+ ...
25
+
26
+ Bail out! test unfinished
27
+ Bail out! test unfinished
@@ -0,0 +1,23 @@
1
+ Test file starting
2
+ TAP version 14
3
+ Invalid host test starting
4
+ Got error: ENOTFOUND
5
+ # Subtest: Connect to an invalid host
6
+ ok 1 - should be equal
7
+ 1..1
8
+ Teardown called
9
+ ok 1 - Connect to an invalid host # time=10.054ms
10
+
11
+ MySQL connection test starting
12
+ Connection object created
13
+ Running query...
14
+ Query issued, waiting for callback...
15
+ Query callback received
16
+ Query result: [ { test: 1 } ]
17
+ Connection destroyed
18
+ # Subtest: Create MySQL connection
19
+ ok 1 - connection worked
20
+ 1..1
21
+ ok 2 - Create MySQL connection # time=11.332ms
22
+
23
+ 1..2
@@ -0,0 +1,35 @@
1
+ Test file starting
2
+ TAP version 14
3
+ Invalid host test starting
4
+ Got error: ENOTFOUND
5
+ # Subtest: Connect to an invalid host
6
+ ok 1 - should be equal
7
+ 1..1
8
+ Teardown called
9
+ ok 1 - Connect to an invalid host # time=9.46ms
10
+
11
+ Init db test starting
12
+ Creating connection...
13
+ Connection created, running querySequence...
14
+ querySequence called, waiting for callback...
15
+ # Subtest: Init db with querySequence
16
+ not ok 1 - test unfinished
17
+ ---
18
+ at:
19
+ fileName: test_debug_init2.js
20
+ lineNumber: 37
21
+ columnNumber: 5
22
+ isToplevel: true
23
+ stack: |
24
+ test_debug_init2.js:37:5
25
+ test: Init db with querySequence
26
+ source: |
27
+ });
28
+
29
+ tap.test('Init db with querySequence', async test => {
30
+ ----^
31
+ console.log('Init db test starting');
32
+ ...
33
+
34
+ Bail out! test unfinished
35
+ Bail out! test unfinished
@@ -0,0 +1,34 @@
1
+ Test file starting at: 1769661151919
2
+ TAP version 14
3
+ Invalid host test starting at: 1769661151922
4
+ ZongJi.start() called at: 1769661151927
5
+ Got error at: 1769661151930 ENOTFOUND
6
+ # Subtest: Connect to an invalid host
7
+ ok 1 - should be equal
8
+ 1..1
9
+ Teardown called at: 1769661151931
10
+ ok 1 - Connect to an invalid host # time=10.183ms
11
+
12
+ Init db test starting at: 1769661151932
13
+ # Subtest: Initialise testing db
14
+ not ok 1 - test unfinished
15
+ ---
16
+ at:
17
+ fileName: test_invalid_host.js
18
+ lineNumber: 30
19
+ columnNumber: 5
20
+ isToplevel: true
21
+ stack: |
22
+ test_invalid_host.js:30:5
23
+ test: Initialise testing db
24
+ source: |
25
+ });
26
+
27
+ tap.test('Initialise testing db', async test => {
28
+ ----^
29
+ console.log('Init db test starting at:', Date.now());
30
+ try {
31
+ ...
32
+
33
+ Bail out! test unfinished
34
+ Bail out! test unfinished
@@ -0,0 +1,26 @@
1
+ Test file starting
2
+ TAP version 14
3
+ Invalid host test starting
4
+ Calling zongji.start()
5
+ zongji.start() returned
6
+ Got error: ENOTFOUND
7
+ # Subtest: Connect to an invalid host
8
+ ok 1 - should be equal
9
+ Calling test.end()
10
+ 1..1
11
+ Teardown starting
12
+ Teardown completed
13
+ ok 1 - Connect to an invalid host # time=10.599ms
14
+
15
+ MySQL test starting
16
+ Calling conn.query()
17
+ conn.query() returned, waiting...
18
+ test.end() returned
19
+ Query callback! err: null
20
+ # Subtest: Simple MySQL query
21
+ ok 1 - query worked
22
+ Timeout reached
23
+ 1..1
24
+ ok 2 - Simple MySQL query # time=5001.569ms
25
+
26
+ 1..2
@@ -0,0 +1,14 @@
1
+ Test file starting
2
+ TAP version 14
3
+ Init db test starting
4
+ Creating connection...
5
+ Connection created, running querySequence...
6
+ querySequence called, waiting for callback...
7
+ querySequence callback received, error: null
8
+ Init complete
9
+ # Subtest: Init db with querySequence
10
+ ok 1 - init worked
11
+ 1..1
12
+ ok 1 - Init db with querySequence # time=18.904ms
13
+
14
+ 1..1
@@ -0,0 +1,17 @@
1
+ Test file starting at: 1769661140028
2
+ TAP version 14
3
+ Test 1 starting at: 1769661140031
4
+ Test 1 ending at: 1769661140132
5
+ # Subtest: Test 1
6
+ ok 1 - test 1 done
7
+ 1..1
8
+ ok 1 - Test 1 # time=103.078ms
9
+
10
+ Test 2 starting at: 1769661140135
11
+ Test 2 init complete at: 1769661140170
12
+ # Subtest: Test 2 - init db
13
+ ok 1 - database initialized
14
+ 1..1
15
+ ok 2 - Test 2 - init db # time=35.205ms
16
+
17
+ 1..2
@@ -0,0 +1,15 @@
1
+ TAP version 14
2
+ # Subtest: Connect to an invalid host - with explicit cleanup
3
+ ok 1 - should be equal
4
+ not ok 2 - getaddrinfo ENOTFOUND wronghost
5
+ ---
6
+ errno: -3008
7
+ code: ENOTFOUND
8
+ syscall: getaddrinfo
9
+ hostname: wronghost
10
+ fatal: true
11
+ tapCaught: unhandledRejection
12
+ ...
13
+
14
+ Bail out! getaddrinfo ENOTFOUND wronghost
15
+ Bail out! getaddrinfo ENOTFOUND wronghost
package/CHANGELOG.md ADDED
@@ -0,0 +1,55 @@
1
+ # Changelog
2
+
3
+ All notable changes to @vlasky/zongji since forking from nevill/zongji.
4
+
5
+ ## [0.6.0] - 2026-02-13
6
+
7
+ - Migrate from @vlasky/mysql to mysql2
8
+ - Convert codebase to ES modules
9
+ - Add TypeScript definitions
10
+ - Add official support for MySQL 8.4
11
+ - Fix sequence ID warnings when using compression with binlog streams
12
+ - Fix connection cleanup in stop() to prevent reuse of destroyed connections
13
+ - Add stopped flag to support dynamic filter updates and safe stop during init
14
+ - Fix flaky error test by handling all error events instead of just the first
15
+
16
+ ## [0.5.9] - 2023-01-11
17
+
18
+ - Allow BLOB columns with utf8mb3 charset
19
+
20
+ ## [0.5.8] - 2021-09-19
21
+
22
+ - Internal version bump
23
+
24
+ ## [0.5.7] - 2021-07-08
25
+
26
+ - Fix connection when binlog_checksum is NONE
27
+
28
+ ## [0.5.6] - 2021-04-30
29
+
30
+ - Update to @vlasky/mysql 2.18.5 with keepalive probe packet support
31
+
32
+ ## [0.5.5] - 2021-04-17
33
+
34
+ - Update to @vlasky/mysql 2.18.4 to support additional charset collations in MySQL 8
35
+
36
+ ## [0.5.4] - 2021-04-17
37
+
38
+ - Update to @vlasky/mysql 2.18.3 to support caching_sha2_password authentication plugin (MySQL 8 default)
39
+
40
+ ## [0.5.3] - 2021-03-24
41
+
42
+ - Update to @vlasky/mysql 2.18.2 to support new MySQL 8 error codes
43
+ - Handle table map events that change table IDs (from YousefED)
44
+ - Fix null value in JSON column causing buffer RangeError (from YousefED)
45
+ - MySQL 8 compatibility fix for column mapping query order
46
+
47
+ ## [0.5.2] - 2020-11-11
48
+
49
+ - Fix IEEE754 conversion error using DataView (from jefbarn)
50
+ - Update dependencies
51
+
52
+ ## [0.5.1] - 2019-11-09
53
+
54
+ - Initial fork from nevill/zongji
55
+ - Add binlog_row_image support
package/CLAUDE.md ADDED
@@ -0,0 +1,99 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Project Overview
6
+
7
+ @vlasky/zongji is a MySQL binlog listener for Node.js that supports MySQL 5.7-8.4. It provides real-time change data capture (CDC) by listening to MySQL binary log events.
8
+
9
+ The library uses `mysql2` for connections and authentication while implementing custom binlog protocol parsing.
10
+
11
+ ## Development Commands
12
+
13
+ ```bash
14
+ # Install dependencies
15
+ npm install
16
+
17
+ # Run zongji tests (requires Docker)
18
+ npm test
19
+
20
+ # Run linter
21
+ npm run lint
22
+
23
+ # Check TypeScript definitions and JS type errors
24
+ npm run checkjs
25
+ ```
26
+
27
+ ## Docker MySQL Client Command
28
+
29
+ - Use this command to launch the mysql client: docker exec -it docker-mysql mysql -u root -psecret
30
+
31
+ ## Architecture Overview
32
+
33
+ The codebase follows an event-driven architecture with these key components:
34
+
35
+ 1. **Main Entry**: `index.js` - ZongJi class extending EventEmitter
36
+ 2. **Dual Connection Pattern**:
37
+ - Control connection for queries/metadata
38
+ - Binlog connection for streaming events
39
+ 3. **Event Processing Pipeline**:
40
+ - Custom `Command` class in `lib/sequence/binlog.js` for mysql2 integration
41
+ - Binary protocol parsing via `Parser` class in `lib/reader.js`
42
+ - Event type definitions in `lib/binlog_event.js`
43
+ - Row event handling in `lib/rows_event.js`
44
+ - Type-specific decoders (datetime, JSON) in lib/
45
+
46
+ ## Key Implementation Details
47
+
48
+ - **Module System**: Uses ES modules (ESM) with import/export syntax
49
+ - **TypeScript Support**: Includes TypeScript definitions (index.d.ts)
50
+ - **MySQL Driver**: Uses `mysql2` for connections, authentication (including caching_sha2_password), and base protocol
51
+ - **Custom Binlog Command**: Implements COM_BINLOG_DUMP (0x12) via custom `Command` class compatible with mysql2's command queue
52
+ - **Parser Compatibility**: Custom `Parser` class in `lib/reader.js` provides mysql2 packet parsing interface
53
+ - **Event Filtering**: Schema and event-type filtering before emission
54
+ - **Position Tracking**: Maintains binlog filename and position for resumable processing
55
+ - **Table Metadata**: Caches table structures for efficient row decoding
56
+ - **Checksum Support**: Automatic detection and handling of binlog checksums (CRC32)
57
+ - **Timezone Support**: Datetime decoders respect mysql2's `timezone` connection option
58
+ - **GTID Support**: Parses Gtid, AnonymousGtid, and PreviousGtids events
59
+
60
+ ## Testing
61
+
62
+ Tests use Docker Compose with MySQL containers:
63
+ - Configuration in `docker-compose.yml`
64
+ - Test database setup in `test/helpers/`
65
+ - Event-specific tests in `test/` directory
66
+ - Uses `tap` test framework (v21+)
67
+
68
+ **Before running tests**, always:
69
+ 1. Ensure Docker containers are running: `docker-compose up -d` (or specific services e.g. `docker-compose up -d mysql80 mysql84`)
70
+ 2. Note: MySQL 5.7 image does not support ARM64 (Apple Silicon) — skip it on those machines
71
+ 3. Wait for MySQL to be ready before running tests (containers need ~10-15 seconds to initialise)
72
+
73
+ When modifying binlog parsing or adding event support, ensure tests cover:
74
+ - Event parsing correctness
75
+ - Position tracking accuracy
76
+ - Error handling scenarios
77
+ - MySQL version compatibility
78
+
79
+ ## Common Development Tasks
80
+
81
+ 1. **Adding new event types**: Update `lib/binlog_event.js` and `lib/code_map.js`
82
+ 2. **Supporting new MySQL types**: Modify type parsing in `lib/rows_event.js` and `lib/common.js`
83
+ 3. **Protocol changes**: Update `lib/sequence/binlog.js` for command handling, `lib/reader.js` for parsing
84
+ 4. **Connection handling**: Modify `index.js` for connection lifecycle changes
85
+
86
+ ## Requirements
87
+
88
+ - Node.js >= 18.0
89
+ - MySQL 5.7, 8.0, or 8.4 with binlog enabled
90
+
91
+ ## Development Principles
92
+
93
+ - **Avoid Duplicate Code**: Avoid creating duplicate code or duplicate functionality
94
+
95
+ # important-instruction-reminders
96
+ Do what has been asked; nothing more, nothing less.
97
+ NEVER create files unless they're absolutely necessary for achieving your goal.
98
+ ALWAYS prefer editing an existing file to creating a new one.
99
+ NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
package/GTID.md ADDED
@@ -0,0 +1,54 @@
1
+ # GTID discussion
2
+
3
+ This file summarizes our discussion about GTID handling in zongji and possible
4
+ exposure patterns for application use.
5
+
6
+ ## Current behavior
7
+
8
+ - GTID-related binlog events are parsed and emitted as their own events:
9
+ - `Gtid`
10
+ - `AnonymousGtid`
11
+ - `PreviousGtids`
12
+ - The GTID event objects contain:
13
+ - `event.gtid` (string: `uuid:sequence`)
14
+ - `event.sid` (UUID string)
15
+ - `event.gno` (sequence number)
16
+ - For `PreviousGtids`: `event.sids` and `event.gtidSet`
17
+ - Row events (`WriteRows`, `UpdateRows`, `DeleteRows`) do not currently include
18
+ GTID information.
19
+
20
+ ## Practical uses for GTID
21
+
22
+ - Checkpointing: store the latest GTID and resume from it after restart/failover.
23
+ - Deduplication / exactly-once: use GTID as a unique transaction ID.
24
+ - Auditing / debugging: correlate binlog events with replication state.
25
+
26
+ ## Transaction boundaries
27
+
28
+ - Start of a GTID transaction:
29
+ - `Gtid` event (when `gtid_mode=ON`).
30
+ - Start of a non-GTID transaction:
31
+ - `Query` event with `BEGIN`.
32
+ - End (commit) of row-based transactions:
33
+ - `Xid` event.
34
+ - End (commit/rollback) of statement-based transactions:
35
+ - `Query` event with `COMMIT` or `ROLLBACK`.
36
+
37
+ ## Open questions / options
38
+
39
+ 1) Add `zongji.lastGtid`:
40
+ - Updated when a `Gtid` or `AnonymousGtid` event is received.
41
+ - Useful for external checkpointing without subscribing to GTID events.
42
+
43
+ 2) Attach `event.gtid` to row events:
44
+ - Copied from `zongji.lastGtid`.
45
+ - Lets users compare row events to see when GTID changes.
46
+ - Only reliable when `gtid_mode=ON` (undefined for non-GTID transactions).
47
+
48
+ 3) Add high-level transaction boundary events:
49
+ - Emit `transactionStart`/`transactionCommit`/`transactionRollback` with
50
+ GTID and event metadata.
51
+
52
+ Notes:
53
+ - Adding `event.gtid` and/or `zongji.lastGtid` is additive and should be
54
+ backward-compatible, but it depends on GTID mode being enabled in MySQL.
package/README.md CHANGED
@@ -1,23 +1,21 @@
1
- A MySQL 8.0-compatible fork of ZongJi - a MySQL binlog listener for Node.js, [originally created by Nevill Dutt](https://github.com/nevill/zongji).
1
+ MySQL binlog-based change data capture (CDC) for Node.js, [originally created by Nevill Dutt](https://github.com/nevill/zongji).
2
2
 
3
- [@vlasky/zongji](https://github.com/vlasky/zongji) has been tested working with MySQL 5.5, 5.6, 5.7 and 8.0.
3
+ [@vlasky/zongji](https://github.com/vlasky/zongji) has been tested working with MySQL 5.7, 8.0 and 8.4.
4
4
 
5
- It leverages [`@vlasky/mysql`](https://github.com/vlasky/mysql), a fork of [`mysql`](https://github.com/mysqljs/mysql) with the following enhancements:
6
-
7
- * Support for authentication using the caching_sha2_password plugin, the new default authentication method in MySQL 8.0
8
- * Partial support for the MySQL compressed protocol (reads compressed data sent by server)
9
- * Optional sending of keepalive probe packets to check the state of the connection to the MySQL server and help keep the connection open when the network socket is idle
5
+ It leverages [`mysql2`](https://github.com/sidorares/node-mysql2) for connections and authentication, while using zongji's binlog parsing and event pipeline.
10
6
 
11
7
  # Latest Release
12
8
 
13
- ZongJi release versions since 0.5.0 only support Node.js version 8 and above.
9
+ Version 0.6.0 is a major modernisation that rewrites the codebase to use the mysql2 module, ES6 syntax and ESM exports, adds TypeScript definitions, and adds official support for MySQL 8.4.
14
10
 
15
- Version 0.4.7 is the last release that supports Node.js version 4.x.
11
+ Version 0.5.9 is the last release that supports Node.js versions below 18 and CommonJS.
16
12
 
17
13
  ## Quick Start
18
14
 
19
15
  ```javascript
20
- let zongji = new ZongJi({ /* ... MySQL Connection Settings ... */ });
16
+ import ZongJi from '@vlasky/zongji';
17
+
18
+ const zongji = new ZongJi({ /* ... MySQL Connection Settings ... */ });
21
19
 
22
20
  // Each change to the replication log results in an event
23
21
  zongji.on('binlog', function(evt) {
@@ -30,18 +28,37 @@ zongji.start({
30
28
  });
31
29
  ```
32
30
 
31
+ ### GTID example
32
+
33
+ ```javascript
34
+ zongji.on('binlog', function(evt) {
35
+ const type = evt.getTypeName();
36
+ if (type === 'Gtid' || type === 'AnonymousGtid') {
37
+ console.log('GTID:', evt.gtid, 'SID:', evt.sid, 'GNO:', evt.gno);
38
+ }
39
+ });
40
+ ```
41
+
42
+ ### TypeScript
43
+
44
+ TypeScript definitions are included:
45
+
46
+ ```typescript
47
+ import ZongJi from '@vlasky/zongji';
48
+ ```
49
+
33
50
  For a complete implementation see [`example.js`](example.js)...
34
51
 
35
52
  ## Installation
36
53
 
37
- * Requires Node.js v8+
54
+ * Requires Node.js v18+
38
55
 
39
56
  ```bash
40
57
  $ npm install @vlasky/zongji
41
58
  ```
42
59
 
43
60
  * Enable MySQL binlog in `my.cnf`, restart MySQL server after making the changes.
44
- > From [MySQL 5.6](https://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html), binlog checksum is enabled by default. Zongji can work with it, but it doesn't really verify it.
61
+ > Binlog checksum is enabled by default in all supported MySQL versions. Zongji can work with it, but it doesn't verify the checksum.
45
62
 
46
63
  ```
47
64
  # Must be unique integer from 1-2^32
@@ -66,12 +83,12 @@ For a complete implementation see [`example.js`](example.js)...
66
83
 
67
84
  The `ZongJi` constructor accepts one argument of either:
68
85
 
69
- * An object containing MySQL connection details in the same format as used by [package mysql](https://npm.im/mysql)
70
- * Or, a [mysql](https://npm.im/mysql) `Connection` or `Pool` object that will be used for querying column information.
86
+ * An object containing MySQL connection details in the same format as used by [package mysql2](https://npm.im/mysql2)
87
+ * Or, a [mysql2](https://npm.im/mysql2) `Connection` or `Pool` object that will be used for querying column information.
71
88
 
72
89
  If a `Connection` or `Pool` object is passed to the constructor, it will not be destroyed/ended by Zongji's `stop()` method.
73
90
 
74
- If there is a `dateStrings` `mysql` configuration option in the connection details or connection, `ZongJi` will follow it.
91
+ The `dateStrings` and `timezone` configuration options from the connection details are respected for date/time value handling.
75
92
 
76
93
  Each instance includes the following methods:
77
94
 
@@ -85,7 +102,7 @@ Some events can be emitted in different phases:
85
102
 
86
103
  Event Name | Description
87
104
  -----------|------------------------
88
- `ready` | This event is occurred right after ZongJi successfully established a connection, setup slave status, and set binlog position.
105
+ `ready` | This event occurs right after ZongJi successfully establishes a connection, sets up replica (slave) status, and sets the binlog position.
89
106
  `binlog` | Once a binlog is received and passes the filter, it will bubble up with this event.
90
107
  `error` | Every error will be caught by this event.
91
108
  `stopped` | Emitted when ZongJi connection is stopped (ZongJi#stop is called).
@@ -116,6 +133,9 @@ Event name | Description
116
133
  `rotate` | [New Binlog file](https://dev.mysql.com/doc/internals/en/rotate-event.html) Not required to be included to rotate to new files, but it is required to be included in order to keep the `filename` and `position` properties updated with current values for [graceful restarting on errors](https://gist.github.com/numtel/5b37b2a7f47b380c1a099596c6f3db2f).
117
134
  `format` | [Format Description](https://dev.mysql.com/doc/internals/en/format-description-event.html)
118
135
  `xid` | [Transaction ID](https://dev.mysql.com/doc/internals/en/xid-event.html)
136
+ `gtid` | GTID event with `gtid`, `sid`, `gno` properties
137
+ `anonymousgtid` | Anonymous GTID event (same shape as `gtid`)
138
+ `previousgtids` | Previous GTIDs event with `gtidSet` and `sids`
119
139
  `tablemap` | Before any row event (must be included for any other row events)
120
140
  `writerows` | Rows inserted, row data array available as `rows` property on event object
121
141
  `updaterows` | Rows changed, row data array available as `rows` property on event object
@@ -132,20 +152,23 @@ Name | Description
132
152
 
133
153
  ## Important Notes
134
154
 
135
- * :star2: [All types allowed by `mysql`](https://github.com/mysqljs/mysql#type-casting) are supported by this package.
155
+ * :star2: All MySQL column types are supported, with type casting similar to [mysql2](https://github.com/sidorares/node-mysql2).
136
156
  * :speak_no_evil: 64-bit integer is supported via package big-integer(see #108). If an integer is within the safe range of JS number (-2^53, 2^53), a Number object will returned, otherwise, will return as String.
137
157
  * :point_right: `TRUNCATE` statement does not cause corresponding `DeleteRows` event. Use unqualified `DELETE FROM` for same effect.
138
- * When using fractional seconds with `DATETIME` and `TIMESTAMP` data types in MySQL > 5.6.4, only millisecond precision is available due to the limit of Javascript's `Date` object.
158
+ * When using fractional seconds with `DATETIME` and `TIMESTAMP` data types, only millisecond precision is available due to the limit of Javascript's `Date` object.
159
+ * Binlog checksums (e.g. `CRC32`) are supported; zongji will detect and ignore the checksum bytes at the end of row events.
139
160
 
140
161
  ## Run Tests
141
162
 
142
- * install [Docker](https://www.docker.com/community-edition#download)
143
- * run `docker-compose up` and then `./docker-test.sh`
163
+ * Install [Docker](https://www.docker.com/community-edition#download)
164
+ * Run `docker-compose up -d` to start MySQL containers
165
+ * Run `npm test` to execute the test suite
144
166
 
145
167
  ## References
146
168
 
147
169
  The following resources provided valuable information that greatly assisted in creating ZongJi:
148
170
 
171
+ * https://github.com/sidorares/node-mysql2
149
172
  * https://github.com/mysqljs/mysql
150
173
  * https://github.com/felixge/faster-than-c/
151
174
  * https://web.archive.org/web/20130117004733/https://intuitive-search.blogspot.co.uk/2011/07/binary-log-api-and-replication-listener.html
@@ -153,7 +176,7 @@ The following resources provided valuable information that greatly assisted in c
153
176
  * https://kkaefer.com/node-cpp-modules/
154
177
  * https://dev.mysql.com/doc/internals/en/replication-protocol.html
155
178
  * https://web.archive.org/web/20200201195450/https://www.cs.wichita.edu/~chang/lecture/cs742/program/how-mysql-c-api.html
156
- * https://github.com/jeremycole/mysql_binlog (Ruby implemenation of MySQL binlog parser)
179
+ * https://github.com/jeremycole/mysql_binlog (Ruby implementation of MySQL binlog parser)
157
180
  * https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html
158
181
 
159
182
  ## License
package/TODO.md ADDED
@@ -0,0 +1,43 @@
1
+ # TODO: MySQL 8 binlog events and partial JSON support
2
+
3
+ This note summarizes the discussion about additional MySQL 8 binlog events and
4
+ partial JSON handling.
5
+
6
+ ## Events not currently supported
7
+
8
+ - PARTIAL_UPDATE_ROWS_EVENT
9
+ - TRANSACTION_PAYLOAD_EVENT
10
+ - HEARTBEAT_LOG_EVENT_V2
11
+
12
+ Currently these events are not mapped in `lib/code_map.js`, so they arrive as
13
+ `Unknown` and are effectively ignored.
14
+
15
+ ## Practical impact
16
+
17
+ - PARTIAL_UPDATE_ROWS_EVENT:
18
+ - Emitted when `binlog_row_value_options=PARTIAL_JSON`.
19
+ - Encodes JSON diffs instead of full JSON values.
20
+ - Only appears if partial JSON is enabled.
21
+ - TRANSACTION_PAYLOAD_EVENT:
22
+ - Used with transaction compression (MySQL 8.0.20+).
23
+ - Wraps multiple events in a compressed payload.
24
+ - Only appears if transaction compression is enabled.
25
+ - HEARTBEAT_LOG_EVENT_V2:
26
+ - Replication keepalive/monitoring; no data changes.
27
+
28
+ ## Possible support approach
29
+
30
+ 1) Add event mappings and parsers:
31
+ - Update `lib/code_map.js` to map event IDs.
32
+ - Implement parser classes in `lib/binlog_event.js` (and any helpers).
33
+ 2) Extend tests:
34
+ - Add targeted tests that enable the corresponding MySQL settings.
35
+ - Assert event emission and basic payload shape.
36
+
37
+ ## Partial JSON strategy options
38
+
39
+ - Option A: expose raw JSON diff data from PARTIAL_UPDATE_ROWS_EVENT.
40
+ - Lowest effort; consumers apply patches.
41
+ - Option B: reconstruct full JSON values.
42
+ - Requires FULL row image or a row-state cache to apply diffs.
43
+ - More complex but produces complete row values.
@@ -1,32 +1,33 @@
1
1
  ---
2
- version: '2'
3
2
  services:
4
- mysql55:
5
- image: mysql:5.5
3
+ mysql57:
4
+ image: mysql:5.7
6
5
  command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
7
6
  networks:
8
7
  default:
9
8
  aliases:
10
- - mysql55
9
+ - mysql57
11
10
  environment:
12
- MYSQL_ROOT_PASSWORD: numtel
11
+ MYSQL_ROOT_PASSWORD: secret
13
12
 
14
- mysql56:
15
- image: mysql:5.6
13
+ mysql80:
14
+ image: mysql:8.0
16
15
  command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
17
16
  networks:
18
17
  default:
19
18
  aliases:
20
- - mysql56
19
+ - mysql80
21
20
  environment:
22
- MYSQL_ROOT_PASSWORD: numtel
21
+ MYSQL_ROOT_PASSWORD: secret
23
22
 
24
- mysql57:
25
- image: mysql:5.7
23
+ mysql84:
24
+ image: mysql:8.4
26
25
  command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
26
+ ports:
27
+ - "3306:3306"
27
28
  networks:
28
29
  default:
29
30
  aliases:
30
- - mysql57
31
+ - mysql84
31
32
  environment:
32
- MYSQL_ROOT_PASSWORD: numtel
33
+ MYSQL_ROOT_PASSWORD: secret