@vlasky/zongji 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +16 -0
- package/.mcp.json +17 -0
- package/.tap/processinfo/84de694d-068b-4ac7-b4d8-f835c2122b2d.json +892 -0
- package/.tap/processinfo/a1e9627e-66eb-412c-aaa5-079a2eb9f385.json +891 -0
- package/.tap/processinfo/abc34c6f-f4aa-4405-8f9a-947048be1df2.json +892 -0
- package/.tap/processinfo/affe6c5b-a34d-41b9-bb73-4b14d35138de.json +891 -0
- package/.tap/processinfo/bf82449d-cdeb-478b-9506-b939a9506bf2.json +883 -0
- package/.tap/processinfo/dd98c508-7067-4342-bdf5-63664b630701.json +892 -0
- package/.tap/test-results/test_await_end.js.tap +19 -0
- package/.tap/test-results/test_await_end2.js.tap +27 -0
- package/.tap/test-results/test_debug_init.js.tap +23 -0
- package/.tap/test-results/test_debug_init2.js.tap +35 -0
- package/.tap/test-results/test_invalid_host.js.tap +34 -0
- package/.tap/test-results/test_minimal.js.tap +26 -0
- package/.tap/test-results/test_queryseq.js.tap +14 -0
- package/.tap/test-results/test_timing.js.tap +17 -0
- package/.tap/test-results/test_unref.js.tap +15 -0
- package/CHANGELOG.md +55 -0
- package/CLAUDE.md +99 -0
- package/GTID.md +54 -0
- package/README.md +44 -21
- package/TODO.md +43 -0
- package/docker-compose.yml +14 -13
- package/docker-test.sh +7 -7
- package/eslint.config.js +35 -0
- package/example.js +1 -1
- package/index.d.ts +314 -0
- package/index.js +444 -272
- package/jsconfig.json +15 -0
- package/lib/binlog_event.js +296 -217
- package/lib/code_map.js +7 -4
- package/lib/common.js +46 -24
- package/lib/datetime_decode.js +147 -34
- package/lib/json_decode.js +12 -6
- package/lib/packet/binlog.js +3 -3
- package/lib/packet/combinlog.js +22 -20
- package/lib/packet/index.js +50 -50
- package/lib/reader.js +215 -109
- package/lib/rows_event.js +98 -102
- package/lib/sequence/binlog.js +142 -39
- package/notes.md +77 -0
- package/package.json +17 -11
- package/scripts/start-mysql.sh +28 -0
- package/workerthreads.md +173 -0
- package/.eslintrc +0 -19
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Test file starting
|
|
2
|
+
TAP version 14
|
|
3
|
+
Invalid host test starting
|
|
4
|
+
Got error: ENOTFOUND
|
|
5
|
+
# Subtest: Connect to an invalid host
|
|
6
|
+
ok 1 - should be equal
|
|
7
|
+
Invalid host test ending
|
|
8
|
+
1..1
|
|
9
|
+
ok 1 - Connect to an invalid host # time=112.602ms
|
|
10
|
+
|
|
11
|
+
MySQL test starting
|
|
12
|
+
Query callback! err: null
|
|
13
|
+
# Subtest: Simple MySQL query
|
|
14
|
+
ok 1 - query worked
|
|
15
|
+
MySQL test ending
|
|
16
|
+
1..1
|
|
17
|
+
ok 2 - Simple MySQL query # time=11.899ms
|
|
18
|
+
|
|
19
|
+
1..2
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
TAP version 14
|
|
2
|
+
# Subtest: Connect to an invalid host
|
|
3
|
+
ok 1 - should be equal
|
|
4
|
+
1..1
|
|
5
|
+
ok 1 - Connect to an invalid host # time=11.133ms
|
|
6
|
+
|
|
7
|
+
# Subtest: Simple MySQL query
|
|
8
|
+
not ok 1 - test unfinished
|
|
9
|
+
---
|
|
10
|
+
at:
|
|
11
|
+
fileName: test_await_end2.js
|
|
12
|
+
lineNumber: 35
|
|
13
|
+
columnNumber: 5
|
|
14
|
+
isToplevel: true
|
|
15
|
+
stack: |
|
|
16
|
+
test_await_end2.js:35:5
|
|
17
|
+
test: Simple MySQL query
|
|
18
|
+
source: |
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
tap.test('Simple MySQL query', async test => {
|
|
22
|
+
----^
|
|
23
|
+
const conn = mysql.createConnection(settings);
|
|
24
|
+
...
|
|
25
|
+
|
|
26
|
+
Bail out! test unfinished
|
|
27
|
+
Bail out! test unfinished
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Test file starting
|
|
2
|
+
TAP version 14
|
|
3
|
+
Invalid host test starting
|
|
4
|
+
Got error: ENOTFOUND
|
|
5
|
+
# Subtest: Connect to an invalid host
|
|
6
|
+
ok 1 - should be equal
|
|
7
|
+
1..1
|
|
8
|
+
Teardown called
|
|
9
|
+
ok 1 - Connect to an invalid host # time=10.054ms
|
|
10
|
+
|
|
11
|
+
MySQL connection test starting
|
|
12
|
+
Connection object created
|
|
13
|
+
Running query...
|
|
14
|
+
Query issued, waiting for callback...
|
|
15
|
+
Query callback received
|
|
16
|
+
Query result: [ { test: 1 } ]
|
|
17
|
+
Connection destroyed
|
|
18
|
+
# Subtest: Create MySQL connection
|
|
19
|
+
ok 1 - connection worked
|
|
20
|
+
1..1
|
|
21
|
+
ok 2 - Create MySQL connection # time=11.332ms
|
|
22
|
+
|
|
23
|
+
1..2
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
Test file starting
|
|
2
|
+
TAP version 14
|
|
3
|
+
Invalid host test starting
|
|
4
|
+
Got error: ENOTFOUND
|
|
5
|
+
# Subtest: Connect to an invalid host
|
|
6
|
+
ok 1 - should be equal
|
|
7
|
+
1..1
|
|
8
|
+
Teardown called
|
|
9
|
+
ok 1 - Connect to an invalid host # time=9.46ms
|
|
10
|
+
|
|
11
|
+
Init db test starting
|
|
12
|
+
Creating connection...
|
|
13
|
+
Connection created, running querySequence...
|
|
14
|
+
querySequence called, waiting for callback...
|
|
15
|
+
# Subtest: Init db with querySequence
|
|
16
|
+
not ok 1 - test unfinished
|
|
17
|
+
---
|
|
18
|
+
at:
|
|
19
|
+
fileName: test_debug_init2.js
|
|
20
|
+
lineNumber: 37
|
|
21
|
+
columnNumber: 5
|
|
22
|
+
isToplevel: true
|
|
23
|
+
stack: |
|
|
24
|
+
test_debug_init2.js:37:5
|
|
25
|
+
test: Init db with querySequence
|
|
26
|
+
source: |
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
tap.test('Init db with querySequence', async test => {
|
|
30
|
+
----^
|
|
31
|
+
console.log('Init db test starting');
|
|
32
|
+
...
|
|
33
|
+
|
|
34
|
+
Bail out! test unfinished
|
|
35
|
+
Bail out! test unfinished
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
Test file starting at: 1769661151919
|
|
2
|
+
TAP version 14
|
|
3
|
+
Invalid host test starting at: 1769661151922
|
|
4
|
+
ZongJi.start() called at: 1769661151927
|
|
5
|
+
Got error at: 1769661151930 ENOTFOUND
|
|
6
|
+
# Subtest: Connect to an invalid host
|
|
7
|
+
ok 1 - should be equal
|
|
8
|
+
1..1
|
|
9
|
+
Teardown called at: 1769661151931
|
|
10
|
+
ok 1 - Connect to an invalid host # time=10.183ms
|
|
11
|
+
|
|
12
|
+
Init db test starting at: 1769661151932
|
|
13
|
+
# Subtest: Initialise testing db
|
|
14
|
+
not ok 1 - test unfinished
|
|
15
|
+
---
|
|
16
|
+
at:
|
|
17
|
+
fileName: test_invalid_host.js
|
|
18
|
+
lineNumber: 30
|
|
19
|
+
columnNumber: 5
|
|
20
|
+
isToplevel: true
|
|
21
|
+
stack: |
|
|
22
|
+
test_invalid_host.js:30:5
|
|
23
|
+
test: Initialise testing db
|
|
24
|
+
source: |
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
tap.test('Initialise testing db', async test => {
|
|
28
|
+
----^
|
|
29
|
+
console.log('Init db test starting at:', Date.now());
|
|
30
|
+
try {
|
|
31
|
+
...
|
|
32
|
+
|
|
33
|
+
Bail out! test unfinished
|
|
34
|
+
Bail out! test unfinished
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Test file starting
|
|
2
|
+
TAP version 14
|
|
3
|
+
Invalid host test starting
|
|
4
|
+
Calling zongji.start()
|
|
5
|
+
zongji.start() returned
|
|
6
|
+
Got error: ENOTFOUND
|
|
7
|
+
# Subtest: Connect to an invalid host
|
|
8
|
+
ok 1 - should be equal
|
|
9
|
+
Calling test.end()
|
|
10
|
+
1..1
|
|
11
|
+
Teardown starting
|
|
12
|
+
Teardown completed
|
|
13
|
+
ok 1 - Connect to an invalid host # time=10.599ms
|
|
14
|
+
|
|
15
|
+
MySQL test starting
|
|
16
|
+
Calling conn.query()
|
|
17
|
+
conn.query() returned, waiting...
|
|
18
|
+
test.end() returned
|
|
19
|
+
Query callback! err: null
|
|
20
|
+
# Subtest: Simple MySQL query
|
|
21
|
+
ok 1 - query worked
|
|
22
|
+
Timeout reached
|
|
23
|
+
1..1
|
|
24
|
+
ok 2 - Simple MySQL query # time=5001.569ms
|
|
25
|
+
|
|
26
|
+
1..2
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Test file starting
|
|
2
|
+
TAP version 14
|
|
3
|
+
Init db test starting
|
|
4
|
+
Creating connection...
|
|
5
|
+
Connection created, running querySequence...
|
|
6
|
+
querySequence called, waiting for callback...
|
|
7
|
+
querySequence callback received, error: null
|
|
8
|
+
Init complete
|
|
9
|
+
# Subtest: Init db with querySequence
|
|
10
|
+
ok 1 - init worked
|
|
11
|
+
1..1
|
|
12
|
+
ok 1 - Init db with querySequence # time=18.904ms
|
|
13
|
+
|
|
14
|
+
1..1
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Test file starting at: 1769661140028
|
|
2
|
+
TAP version 14
|
|
3
|
+
Test 1 starting at: 1769661140031
|
|
4
|
+
Test 1 ending at: 1769661140132
|
|
5
|
+
# Subtest: Test 1
|
|
6
|
+
ok 1 - test 1 done
|
|
7
|
+
1..1
|
|
8
|
+
ok 1 - Test 1 # time=103.078ms
|
|
9
|
+
|
|
10
|
+
Test 2 starting at: 1769661140135
|
|
11
|
+
Test 2 init complete at: 1769661140170
|
|
12
|
+
# Subtest: Test 2 - init db
|
|
13
|
+
ok 1 - database initialized
|
|
14
|
+
1..1
|
|
15
|
+
ok 2 - Test 2 - init db # time=35.205ms
|
|
16
|
+
|
|
17
|
+
1..2
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
TAP version 14
|
|
2
|
+
# Subtest: Connect to an invalid host - with explicit cleanup
|
|
3
|
+
ok 1 - should be equal
|
|
4
|
+
not ok 2 - getaddrinfo ENOTFOUND wronghost
|
|
5
|
+
---
|
|
6
|
+
errno: -3008
|
|
7
|
+
code: ENOTFOUND
|
|
8
|
+
syscall: getaddrinfo
|
|
9
|
+
hostname: wronghost
|
|
10
|
+
fatal: true
|
|
11
|
+
tapCaught: unhandledRejection
|
|
12
|
+
...
|
|
13
|
+
|
|
14
|
+
Bail out! getaddrinfo ENOTFOUND wronghost
|
|
15
|
+
Bail out! getaddrinfo ENOTFOUND wronghost
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to @vlasky/zongji since forking from nevill/zongji.
|
|
4
|
+
|
|
5
|
+
## [0.6.0] - 2026-02-13
|
|
6
|
+
|
|
7
|
+
- Migrate from @vlasky/mysql to mysql2
|
|
8
|
+
- Convert codebase to ES modules
|
|
9
|
+
- Add TypeScript definitions
|
|
10
|
+
- Add official support for MySQL 8.4
|
|
11
|
+
- Fix sequence ID warnings when using compression with binlog streams
|
|
12
|
+
- Fix connection cleanup in stop() to prevent reuse of destroyed connections
|
|
13
|
+
- Add stopped flag to support dynamic filter updates and safe stop during init
|
|
14
|
+
- Fix flaky error test by handling all error events instead of just the first
|
|
15
|
+
|
|
16
|
+
## [0.5.9] - 2023-01-11
|
|
17
|
+
|
|
18
|
+
- Allow BLOB columns with utf8mb3 charset
|
|
19
|
+
|
|
20
|
+
## [0.5.8] - 2021-09-19
|
|
21
|
+
|
|
22
|
+
- Internal version bump
|
|
23
|
+
|
|
24
|
+
## [0.5.7] - 2021-07-08
|
|
25
|
+
|
|
26
|
+
- Fix connection when binlog_checksum is NONE
|
|
27
|
+
|
|
28
|
+
## [0.5.6] - 2021-04-30
|
|
29
|
+
|
|
30
|
+
- Update to @vlasky/mysql 2.18.5 with keepalive probe packet support
|
|
31
|
+
|
|
32
|
+
## [0.5.5] - 2021-04-17
|
|
33
|
+
|
|
34
|
+
- Update to @vlasky/mysql 2.18.4 to support additional charset collations in MySQL 8
|
|
35
|
+
|
|
36
|
+
## [0.5.4] - 2021-04-17
|
|
37
|
+
|
|
38
|
+
- Update to @vlasky/mysql 2.18.3 to support caching_sha2_password authentication plugin (MySQL 8 default)
|
|
39
|
+
|
|
40
|
+
## [0.5.3] - 2021-03-24
|
|
41
|
+
|
|
42
|
+
- Update to @vlasky/mysql 2.18.2 to support new MySQL 8 error codes
|
|
43
|
+
- Handle table map events that change table IDs (from YousefED)
|
|
44
|
+
- Fix null value in JSON column causing buffer RangeError (from YousefED)
|
|
45
|
+
- MySQL 8 compatibility fix for column mapping query order
|
|
46
|
+
|
|
47
|
+
## [0.5.2] - 2020-11-11
|
|
48
|
+
|
|
49
|
+
- Fix IEEE754 conversion error using DataView (from jefbarn)
|
|
50
|
+
- Update dependencies
|
|
51
|
+
|
|
52
|
+
## [0.5.1] - 2019-11-09
|
|
53
|
+
|
|
54
|
+
- Initial fork from nevill/zongji
|
|
55
|
+
- Add binlog_row_image support
|
package/CLAUDE.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project Overview
|
|
6
|
+
|
|
7
|
+
@vlasky/zongji is a MySQL binlog listener for Node.js that supports MySQL 5.7-8.4. It provides real-time change data capture (CDC) by listening to MySQL binary log events.
|
|
8
|
+
|
|
9
|
+
The library uses `mysql2` for connections and authentication while implementing custom binlog protocol parsing.
|
|
10
|
+
|
|
11
|
+
## Development Commands
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
# Install dependencies
|
|
15
|
+
npm install
|
|
16
|
+
|
|
17
|
+
# Run zongji tests (requires Docker)
|
|
18
|
+
npm test
|
|
19
|
+
|
|
20
|
+
# Run linter
|
|
21
|
+
npm run lint
|
|
22
|
+
|
|
23
|
+
# Check TypeScript definitions and JS type errors
|
|
24
|
+
npm run checkjs
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Docker MySQL Client Command
|
|
28
|
+
|
|
29
|
+
- Use this command to launch the mysql client: docker exec -it docker-mysql mysql -u root -psecret
|
|
30
|
+
|
|
31
|
+
## Architecture Overview
|
|
32
|
+
|
|
33
|
+
The codebase follows an event-driven architecture with these key components:
|
|
34
|
+
|
|
35
|
+
1. **Main Entry**: `index.js` - ZongJi class extending EventEmitter
|
|
36
|
+
2. **Dual Connection Pattern**:
|
|
37
|
+
- Control connection for queries/metadata
|
|
38
|
+
- Binlog connection for streaming events
|
|
39
|
+
3. **Event Processing Pipeline**:
|
|
40
|
+
- Custom `Command` class in `lib/sequence/binlog.js` for mysql2 integration
|
|
41
|
+
- Binary protocol parsing via `Parser` class in `lib/reader.js`
|
|
42
|
+
- Event type definitions in `lib/binlog_event.js`
|
|
43
|
+
- Row event handling in `lib/rows_event.js`
|
|
44
|
+
- Type-specific decoders (datetime, JSON) in lib/
|
|
45
|
+
|
|
46
|
+
## Key Implementation Details
|
|
47
|
+
|
|
48
|
+
- **Module System**: Uses ES modules (ESM) with import/export syntax
|
|
49
|
+
- **TypeScript Support**: Includes TypeScript definitions (index.d.ts)
|
|
50
|
+
- **MySQL Driver**: Uses `mysql2` for connections, authentication (including caching_sha2_password), and base protocol
|
|
51
|
+
- **Custom Binlog Command**: Implements COM_BINLOG_DUMP (0x12) via custom `Command` class compatible with mysql2's command queue
|
|
52
|
+
- **Parser Compatibility**: Custom `Parser` class in `lib/reader.js` provides mysql2 packet parsing interface
|
|
53
|
+
- **Event Filtering**: Schema and event-type filtering before emission
|
|
54
|
+
- **Position Tracking**: Maintains binlog filename and position for resumable processing
|
|
55
|
+
- **Table Metadata**: Caches table structures for efficient row decoding
|
|
56
|
+
- **Checksum Support**: Automatic detection and handling of binlog checksums (CRC32)
|
|
57
|
+
- **Timezone Support**: Datetime decoders respect mysql2's `timezone` connection option
|
|
58
|
+
- **GTID Support**: Parses Gtid, AnonymousGtid, and PreviousGtids events
|
|
59
|
+
|
|
60
|
+
## Testing
|
|
61
|
+
|
|
62
|
+
Tests use Docker Compose with MySQL containers:
|
|
63
|
+
- Configuration in `docker-compose.yml`
|
|
64
|
+
- Test database setup in `test/helpers/`
|
|
65
|
+
- Event-specific tests in `test/` directory
|
|
66
|
+
- Uses `tap` test framework (v21+)
|
|
67
|
+
|
|
68
|
+
**Before running tests**, always:
|
|
69
|
+
1. Ensure Docker containers are running: `docker-compose up -d` (or specific services e.g. `docker-compose up -d mysql80 mysql84`)
|
|
70
|
+
2. Note: MySQL 5.7 image does not support ARM64 (Apple Silicon) — skip it on those machines
|
|
71
|
+
3. Wait for MySQL to be ready before running tests (containers need ~10-15 seconds to initialise)
|
|
72
|
+
|
|
73
|
+
When modifying binlog parsing or adding event support, ensure tests cover:
|
|
74
|
+
- Event parsing correctness
|
|
75
|
+
- Position tracking accuracy
|
|
76
|
+
- Error handling scenarios
|
|
77
|
+
- MySQL version compatibility
|
|
78
|
+
|
|
79
|
+
## Common Development Tasks
|
|
80
|
+
|
|
81
|
+
1. **Adding new event types**: Update `lib/binlog_event.js` and `lib/code_map.js`
|
|
82
|
+
2. **Supporting new MySQL types**: Modify type parsing in `lib/rows_event.js` and `lib/common.js`
|
|
83
|
+
3. **Protocol changes**: Update `lib/sequence/binlog.js` for command handling, `lib/reader.js` for parsing
|
|
84
|
+
4. **Connection handling**: Modify `index.js` for connection lifecycle changes
|
|
85
|
+
|
|
86
|
+
## Requirements
|
|
87
|
+
|
|
88
|
+
- Node.js >= 18.0
|
|
89
|
+
- MySQL 5.7, 8.0, or 8.4 with binlog enabled
|
|
90
|
+
|
|
91
|
+
## Development Principles
|
|
92
|
+
|
|
93
|
+
- **Avoid Duplicate Code**: Avoid creating duplicate code or duplicate functionality
|
|
94
|
+
|
|
95
|
+
# important-instruction-reminders
|
|
96
|
+
Do what has been asked; nothing more, nothing less.
|
|
97
|
+
NEVER create files unless they're absolutely necessary for achieving your goal.
|
|
98
|
+
ALWAYS prefer editing an existing file to creating a new one.
|
|
99
|
+
NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
|
package/GTID.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# GTID discussion
|
|
2
|
+
|
|
3
|
+
This file summarizes our discussion about GTID handling in zongji and possible
|
|
4
|
+
exposure patterns for application use.
|
|
5
|
+
|
|
6
|
+
## Current behavior
|
|
7
|
+
|
|
8
|
+
- GTID-related binlog events are parsed and emitted as their own events:
|
|
9
|
+
- `Gtid`
|
|
10
|
+
- `AnonymousGtid`
|
|
11
|
+
- `PreviousGtids`
|
|
12
|
+
- The GTID event objects contain:
|
|
13
|
+
- `event.gtid` (string: `uuid:sequence`)
|
|
14
|
+
- `event.sid` (UUID string)
|
|
15
|
+
- `event.gno` (sequence number)
|
|
16
|
+
- For `PreviousGtids`: `event.sids` and `event.gtidSet`
|
|
17
|
+
- Row events (`WriteRows`, `UpdateRows`, `DeleteRows`) do not currently include
|
|
18
|
+
GTID information.
|
|
19
|
+
|
|
20
|
+
## Practical uses for GTID
|
|
21
|
+
|
|
22
|
+
- Checkpointing: store the latest GTID and resume from it after restart/failover.
|
|
23
|
+
- Deduplication / exactly-once: use GTID as a unique transaction ID.
|
|
24
|
+
- Auditing / debugging: correlate binlog events with replication state.
|
|
25
|
+
|
|
26
|
+
## Transaction boundaries
|
|
27
|
+
|
|
28
|
+
- Start of a GTID transaction:
|
|
29
|
+
- `Gtid` event (when `gtid_mode=ON`).
|
|
30
|
+
- Start of a non-GTID transaction:
|
|
31
|
+
- `Query` event with `BEGIN`.
|
|
32
|
+
- End (commit) of row-based transactions:
|
|
33
|
+
- `Xid` event.
|
|
34
|
+
- End (commit/rollback) of statement-based transactions:
|
|
35
|
+
- `Query` event with `COMMIT` or `ROLLBACK`.
|
|
36
|
+
|
|
37
|
+
## Open questions / options
|
|
38
|
+
|
|
39
|
+
1) Add `zongji.lastGtid`:
|
|
40
|
+
- Updated when a `Gtid` or `AnonymousGtid` event is received.
|
|
41
|
+
- Useful for external checkpointing without subscribing to GTID events.
|
|
42
|
+
|
|
43
|
+
2) Attach `event.gtid` to row events:
|
|
44
|
+
- Copied from `zongji.lastGtid`.
|
|
45
|
+
- Lets users compare row events to see when GTID changes.
|
|
46
|
+
- Only reliable when `gtid_mode=ON` (undefined for non-GTID transactions).
|
|
47
|
+
|
|
48
|
+
3) Add high-level transaction boundary events:
|
|
49
|
+
- Emit `transactionStart`/`transactionCommit`/`transactionRollback` with
|
|
50
|
+
GTID and event metadata.
|
|
51
|
+
|
|
52
|
+
Notes:
|
|
53
|
+
- Adding `event.gtid` and/or `zongji.lastGtid` is additive and should be
|
|
54
|
+
backward-compatible, but it depends on GTID mode being enabled in MySQL.
|
package/README.md
CHANGED
|
@@ -1,23 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
MySQL binlog-based change data capture (CDC) for Node.js, [originally created by Nevill Dutt](https://github.com/nevill/zongji).
|
|
2
2
|
|
|
3
|
-
[@vlasky/zongji](https://github.com/vlasky/zongji) has been tested working with MySQL 5.
|
|
3
|
+
[@vlasky/zongji](https://github.com/vlasky/zongji) has been tested working with MySQL 5.7, 8.0 and 8.4.
|
|
4
4
|
|
|
5
|
-
It leverages [
|
|
6
|
-
|
|
7
|
-
* Support for authentication using the caching_sha2_password plugin, the new default authentication method in MySQL 8.0
|
|
8
|
-
* Partial support for the MySQL compressed protocol (reads compressed data sent by server)
|
|
9
|
-
* Optional sending of keepalive probe packets to check the state of the connection to the MySQL server and help keep the connection open when the network socket is idle
|
|
5
|
+
It leverages [`mysql2`](https://github.com/sidorares/node-mysql2) for connections and authentication, while using zongji's binlog parsing and event pipeline.
|
|
10
6
|
|
|
11
7
|
# Latest Release
|
|
12
8
|
|
|
13
|
-
|
|
9
|
+
Version 0.6.0 is a major modernisation that rewrites the codebase to use the mysql2 module, ES6 syntax and ESM exports, adds TypeScript definitions, and adds official support for MySQL 8.4.
|
|
14
10
|
|
|
15
|
-
Version 0.
|
|
11
|
+
Version 0.5.9 is the last release that supports Node.js versions below 18 and CommonJS.
|
|
16
12
|
|
|
17
13
|
## Quick Start
|
|
18
14
|
|
|
19
15
|
```javascript
|
|
20
|
-
|
|
16
|
+
import ZongJi from '@vlasky/zongji';
|
|
17
|
+
|
|
18
|
+
const zongji = new ZongJi({ /* ... MySQL Connection Settings ... */ });
|
|
21
19
|
|
|
22
20
|
// Each change to the replication log results in an event
|
|
23
21
|
zongji.on('binlog', function(evt) {
|
|
@@ -30,18 +28,37 @@ zongji.start({
|
|
|
30
28
|
});
|
|
31
29
|
```
|
|
32
30
|
|
|
31
|
+
### GTID example
|
|
32
|
+
|
|
33
|
+
```javascript
|
|
34
|
+
zongji.on('binlog', function(evt) {
|
|
35
|
+
const type = evt.getTypeName();
|
|
36
|
+
if (type === 'Gtid' || type === 'AnonymousGtid') {
|
|
37
|
+
console.log('GTID:', evt.gtid, 'SID:', evt.sid, 'GNO:', evt.gno);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### TypeScript
|
|
43
|
+
|
|
44
|
+
TypeScript definitions are included:
|
|
45
|
+
|
|
46
|
+
```typescript
|
|
47
|
+
import ZongJi from '@vlasky/zongji';
|
|
48
|
+
```
|
|
49
|
+
|
|
33
50
|
For a complete implementation see [`example.js`](example.js)...
|
|
34
51
|
|
|
35
52
|
## Installation
|
|
36
53
|
|
|
37
|
-
* Requires Node.js
|
|
54
|
+
* Requires Node.js v18+
|
|
38
55
|
|
|
39
56
|
```bash
|
|
40
57
|
$ npm install @vlasky/zongji
|
|
41
58
|
```
|
|
42
59
|
|
|
43
60
|
* Enable MySQL binlog in `my.cnf`, restart MySQL server after making the changes.
|
|
44
|
-
>
|
|
61
|
+
> Binlog checksum is enabled by default in all supported MySQL versions. Zongji can work with it, but it doesn't verify the checksum.
|
|
45
62
|
|
|
46
63
|
```
|
|
47
64
|
# Must be unique integer from 1-2^32
|
|
@@ -66,12 +83,12 @@ For a complete implementation see [`example.js`](example.js)...
|
|
|
66
83
|
|
|
67
84
|
The `ZongJi` constructor accepts one argument of either:
|
|
68
85
|
|
|
69
|
-
* An object containing MySQL connection details in the same format as used by [package
|
|
70
|
-
* Or, a [
|
|
86
|
+
* An object containing MySQL connection details in the same format as used by [package mysql2](https://npm.im/mysql2)
|
|
87
|
+
* Or, a [mysql2](https://npm.im/mysql2) `Connection` or `Pool` object that will be used for querying column information.
|
|
71
88
|
|
|
72
89
|
If a `Connection` or `Pool` object is passed to the constructor, it will not be destroyed/ended by Zongji's `stop()` method.
|
|
73
90
|
|
|
74
|
-
|
|
91
|
+
The `dateStrings` and `timezone` configuration options from the connection details are respected for date/time value handling.
|
|
75
92
|
|
|
76
93
|
Each instance includes the following methods:
|
|
77
94
|
|
|
@@ -85,7 +102,7 @@ Some events can be emitted in different phases:
|
|
|
85
102
|
|
|
86
103
|
Event Name | Description
|
|
87
104
|
-----------|------------------------
|
|
88
|
-
`ready` | This event
|
|
105
|
+
`ready` | This event occurs right after ZongJi successfully establishes a connection, sets up replica (slave) status, and sets the binlog position.
|
|
89
106
|
`binlog` | Once a binlog is received and passes the filter, it will bubble up with this event.
|
|
90
107
|
`error` | Every error will be caught by this event.
|
|
91
108
|
`stopped` | Emitted when ZongJi connection is stopped (ZongJi#stop is called).
|
|
@@ -116,6 +133,9 @@ Event name | Description
|
|
|
116
133
|
`rotate` | [New Binlog file](https://dev.mysql.com/doc/internals/en/rotate-event.html) Not required to be included to rotate to new files, but it is required to be included in order to keep the `filename` and `position` properties updated with current values for [graceful restarting on errors](https://gist.github.com/numtel/5b37b2a7f47b380c1a099596c6f3db2f).
|
|
117
134
|
`format` | [Format Description](https://dev.mysql.com/doc/internals/en/format-description-event.html)
|
|
118
135
|
`xid` | [Transaction ID](https://dev.mysql.com/doc/internals/en/xid-event.html)
|
|
136
|
+
`gtid` | GTID event with `gtid`, `sid`, `gno` properties
|
|
137
|
+
`anonymousgtid` | Anonymous GTID event (same shape as `gtid`)
|
|
138
|
+
`previousgtids` | Previous GTIDs event with `gtidSet` and `sids`
|
|
119
139
|
`tablemap` | Before any row event (must be included for any other row events)
|
|
120
140
|
`writerows` | Rows inserted, row data array available as `rows` property on event object
|
|
121
141
|
`updaterows` | Rows changed, row data array available as `rows` property on event object
|
|
@@ -132,20 +152,23 @@ Name | Description
|
|
|
132
152
|
|
|
133
153
|
## Important Notes
|
|
134
154
|
|
|
135
|
-
* :star2:
|
|
155
|
+
* :star2: All MySQL column types are supported, with type casting similar to [mysql2](https://github.com/sidorares/node-mysql2).
|
|
136
156
|
* :speak_no_evil: 64-bit integer is supported via package big-integer(see #108). If an integer is within the safe range of JS number (-2^53, 2^53), a Number object will returned, otherwise, will return as String.
|
|
137
157
|
* :point_right: `TRUNCATE` statement does not cause corresponding `DeleteRows` event. Use unqualified `DELETE FROM` for same effect.
|
|
138
|
-
* When using fractional seconds with `DATETIME` and `TIMESTAMP` data types
|
|
158
|
+
* When using fractional seconds with `DATETIME` and `TIMESTAMP` data types, only millisecond precision is available due to the limit of Javascript's `Date` object.
|
|
159
|
+
* Binlog checksums (e.g. `CRC32`) are supported; zongji will detect and ignore the checksum bytes at the end of row events.
|
|
139
160
|
|
|
140
161
|
## Run Tests
|
|
141
162
|
|
|
142
|
-
*
|
|
143
|
-
*
|
|
163
|
+
* Install [Docker](https://www.docker.com/community-edition#download)
|
|
164
|
+
* Run `docker-compose up -d` to start MySQL containers
|
|
165
|
+
* Run `npm test` to execute the test suite
|
|
144
166
|
|
|
145
167
|
## References
|
|
146
168
|
|
|
147
169
|
The following resources provided valuable information that greatly assisted in creating ZongJi:
|
|
148
170
|
|
|
171
|
+
* https://github.com/sidorares/node-mysql2
|
|
149
172
|
* https://github.com/mysqljs/mysql
|
|
150
173
|
* https://github.com/felixge/faster-than-c/
|
|
151
174
|
* https://web.archive.org/web/20130117004733/https://intuitive-search.blogspot.co.uk/2011/07/binary-log-api-and-replication-listener.html
|
|
@@ -153,7 +176,7 @@ The following resources provided valuable information that greatly assisted in c
|
|
|
153
176
|
* https://kkaefer.com/node-cpp-modules/
|
|
154
177
|
* https://dev.mysql.com/doc/internals/en/replication-protocol.html
|
|
155
178
|
* https://web.archive.org/web/20200201195450/https://www.cs.wichita.edu/~chang/lecture/cs742/program/how-mysql-c-api.html
|
|
156
|
-
* https://github.com/jeremycole/mysql_binlog (Ruby
|
|
179
|
+
* https://github.com/jeremycole/mysql_binlog (Ruby implementation of MySQL binlog parser)
|
|
157
180
|
* https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html
|
|
158
181
|
|
|
159
182
|
## License
|
package/TODO.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# TODO: MySQL 8 binlog events and partial JSON support
|
|
2
|
+
|
|
3
|
+
This note summarizes the discussion about additional MySQL 8 binlog events and
|
|
4
|
+
partial JSON handling.
|
|
5
|
+
|
|
6
|
+
## Events not currently supported
|
|
7
|
+
|
|
8
|
+
- PARTIAL_UPDATE_ROWS_EVENT
|
|
9
|
+
- TRANSACTION_PAYLOAD_EVENT
|
|
10
|
+
- HEARTBEAT_LOG_EVENT_V2
|
|
11
|
+
|
|
12
|
+
Currently these events are not mapped in `lib/code_map.js`, so they arrive as
|
|
13
|
+
`Unknown` and are effectively ignored.
|
|
14
|
+
|
|
15
|
+
## Practical impact
|
|
16
|
+
|
|
17
|
+
- PARTIAL_UPDATE_ROWS_EVENT:
|
|
18
|
+
- Emitted when `binlog_row_value_options=PARTIAL_JSON`.
|
|
19
|
+
- Encodes JSON diffs instead of full JSON values.
|
|
20
|
+
- Only appears if partial JSON is enabled.
|
|
21
|
+
- TRANSACTION_PAYLOAD_EVENT:
|
|
22
|
+
- Used with transaction compression (MySQL 8.0.20+).
|
|
23
|
+
- Wraps multiple events in a compressed payload.
|
|
24
|
+
- Only appears if transaction compression is enabled.
|
|
25
|
+
- HEARTBEAT_LOG_EVENT_V2:
|
|
26
|
+
- Replication keepalive/monitoring; no data changes.
|
|
27
|
+
|
|
28
|
+
## Possible support approach
|
|
29
|
+
|
|
30
|
+
1) Add event mappings and parsers:
|
|
31
|
+
- Update `lib/code_map.js` to map event IDs.
|
|
32
|
+
- Implement parser classes in `lib/binlog_event.js` (and any helpers).
|
|
33
|
+
2) Extend tests:
|
|
34
|
+
- Add targeted tests that enable the corresponding MySQL settings.
|
|
35
|
+
- Assert event emission and basic payload shape.
|
|
36
|
+
|
|
37
|
+
## Partial JSON strategy options
|
|
38
|
+
|
|
39
|
+
- Option A: expose raw JSON diff data from PARTIAL_UPDATE_ROWS_EVENT.
|
|
40
|
+
- Lowest effort; consumers apply patches.
|
|
41
|
+
- Option B: reconstruct full JSON values.
|
|
42
|
+
- Requires FULL row image or a row-state cache to apply diffs.
|
|
43
|
+
- More complex but produces complete row values.
|
package/docker-compose.yml
CHANGED
|
@@ -1,32 +1,33 @@
|
|
|
1
1
|
---
|
|
2
|
-
version: '2'
|
|
3
2
|
services:
|
|
4
|
-
|
|
5
|
-
image: mysql:5.
|
|
3
|
+
mysql57:
|
|
4
|
+
image: mysql:5.7
|
|
6
5
|
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
7
6
|
networks:
|
|
8
7
|
default:
|
|
9
8
|
aliases:
|
|
10
|
-
-
|
|
9
|
+
- mysql57
|
|
11
10
|
environment:
|
|
12
|
-
MYSQL_ROOT_PASSWORD:
|
|
11
|
+
MYSQL_ROOT_PASSWORD: secret
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
image: mysql:
|
|
13
|
+
mysql80:
|
|
14
|
+
image: mysql:8.0
|
|
16
15
|
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
17
16
|
networks:
|
|
18
17
|
default:
|
|
19
18
|
aliases:
|
|
20
|
-
-
|
|
19
|
+
- mysql80
|
|
21
20
|
environment:
|
|
22
|
-
MYSQL_ROOT_PASSWORD:
|
|
21
|
+
MYSQL_ROOT_PASSWORD: secret
|
|
23
22
|
|
|
24
|
-
|
|
25
|
-
image: mysql:
|
|
23
|
+
mysql84:
|
|
24
|
+
image: mysql:8.4
|
|
26
25
|
command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
|
|
26
|
+
ports:
|
|
27
|
+
- "3306:3306"
|
|
27
28
|
networks:
|
|
28
29
|
default:
|
|
29
30
|
aliases:
|
|
30
|
-
-
|
|
31
|
+
- mysql84
|
|
31
32
|
environment:
|
|
32
|
-
MYSQL_ROOT_PASSWORD:
|
|
33
|
+
MYSQL_ROOT_PASSWORD: secret
|