fluent-plugin-mysql-replicator 1.0.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.devcontainer/Dockerfile +17 -0
- data/.devcontainer/devcontainer.json +16 -0
- data/.dockerignore +4 -0
- data/.github/workflows/ci.yml +165 -0
- data/.gitignore +1 -0
- data/CHANGELOG.md +90 -0
- data/README.md +80 -5
- data/Tutorial-mysql_replicator.md +21 -1
- data/ci/install_smoke.sh +19 -0
- data/ci/verify_plugins_load.rb +5 -0
- data/fluent-plugin-mysql-replicator.gemspec +1 -1
- data/lib/fluent/plugin/in_mysql_replicator.rb +47 -9
- data/lib/fluent/plugin/in_mysql_replicator_multi.rb +19 -0
- data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb +44 -7
- data/setup_mysql_replicator_multi.sql +3 -0
- data/test/e2e/e2e_helper.rb +94 -0
- data/test/e2e/fluent_multi.conf +30 -0
- data/test/e2e/fluent_single.conf +29 -0
- data/test/e2e/replication_multi_e2e_test.rb +140 -0
- data/test/e2e/replication_single_e2e_test.rb +91 -0
- data/test/plugin/test_in_mysql_replicator.rb +100 -0
- data/test/plugin/test_out_mysql_replicator_elasticsearch.rb +22 -0
- metadata +20 -4
- data/.travis.yml +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ea8d93c147df39ff2c34ac573a5e981c1e0a46da8445395eaf38cdc44abf7c0e
|
|
4
|
+
data.tar.gz: f190b88e93c4318b9c2ec42863186f7dc1f0a3a911bb60e2b689a84687bffdcd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bfbf04041525c768d56c7b0bb3d2cd69ca23076eca57d162277d81a06c1cb4290e4b0d402e46e4b136ebf2d45b8fdca59555c1406c9763336b67f3cf2b4be35d
|
|
7
|
+
data.tar.gz: 2872f2e0d15084d3680a77498411aa5165acb000b2b5744be19686a2dcd89b41ccee8169956203bb3b9c9a004b0aba6d16d741147215b518e0690889b220ab98
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
FROM ruby:3.4-slim
|
|
2
|
+
|
|
3
|
+
ENV DEBIAN_FRONTEND=noninteractive
|
|
4
|
+
ENV BUNDLE_PATH=/workspace/vendor/bundle
|
|
5
|
+
|
|
6
|
+
RUN apt-get update \
|
|
7
|
+
&& apt-get install -y --no-install-recommends \
|
|
8
|
+
build-essential \
|
|
9
|
+
default-libmysqlclient-dev \
|
|
10
|
+
git \
|
|
11
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
12
|
+
|
|
13
|
+
WORKDIR /workspace
|
|
14
|
+
|
|
15
|
+
RUN gem install bundler
|
|
16
|
+
|
|
17
|
+
CMD ["bash"]
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "fluent-plugin-mysql-replicator",
|
|
3
|
+
"build": {
|
|
4
|
+
"dockerfile": "Dockerfile"
|
|
5
|
+
},
|
|
6
|
+
"workspaceFolder": "/workspace",
|
|
7
|
+
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
|
|
8
|
+
"settings": {
|
|
9
|
+
"terminal.integrated.shell.linux": "/bin/bash"
|
|
10
|
+
},
|
|
11
|
+
"extensions": [
|
|
12
|
+
"rebornix.Ruby"
|
|
13
|
+
],
|
|
14
|
+
"postCreateCommand": "bundle config set --local path \"$BUNDLE_PATH\" && bundle install --jobs 4 --retry 3",
|
|
15
|
+
"remoteUser": "root"
|
|
16
|
+
}
|
data/.dockerignore
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
unit:
|
|
10
|
+
name: Unit tests (Ruby ${{ matrix.ruby }})
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
ruby: ['3.2', '3.3', '3.4', '4.0']
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v5
|
|
18
|
+
- name: Install MySQL client headers
|
|
19
|
+
run: sudo apt-get update && sudo apt-get install -y default-libmysqlclient-dev
|
|
20
|
+
- uses: ruby/setup-ruby@v1
|
|
21
|
+
with:
|
|
22
|
+
ruby-version: ${{ matrix.ruby }}
|
|
23
|
+
bundler-cache: true
|
|
24
|
+
- run: bundle exec rake test
|
|
25
|
+
|
|
26
|
+
e2e:
|
|
27
|
+
name: E2E replication (ES ${{ matrix.es_label }})
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
include:
|
|
34
|
+
# The plugin auto-detects the version and adjusts "_type" accordingly.
|
|
35
|
+
- es_label: '6.8'
|
|
36
|
+
es_image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
|
|
37
|
+
es_major: '6'
|
|
38
|
+
- es_label: '8.x'
|
|
39
|
+
es_image: docker.elastic.co/elasticsearch/elasticsearch:8.18.0
|
|
40
|
+
es_major: '8'
|
|
41
|
+
- es_label: '9.x'
|
|
42
|
+
es_image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0
|
|
43
|
+
es_major: '9'
|
|
44
|
+
|
|
45
|
+
services:
|
|
46
|
+
mysql:
|
|
47
|
+
image: mysql:8.4
|
|
48
|
+
env:
|
|
49
|
+
MYSQL_ROOT_PASSWORD: root
|
|
50
|
+
ports:
|
|
51
|
+
- 3306:3306
|
|
52
|
+
options: >-
|
|
53
|
+
--health-cmd="mysqladmin ping -h localhost -proot"
|
|
54
|
+
--health-interval=10s
|
|
55
|
+
--health-timeout=5s
|
|
56
|
+
--health-retries=10
|
|
57
|
+
|
|
58
|
+
elasticsearch:
|
|
59
|
+
image: ${{ matrix.es_image }}
|
|
60
|
+
env:
|
|
61
|
+
discovery.type: single-node
|
|
62
|
+
xpack.security.enabled: 'false'
|
|
63
|
+
ES_JAVA_OPTS: -Xms512m -Xmx512m
|
|
64
|
+
ports:
|
|
65
|
+
- 9200:9200
|
|
66
|
+
|
|
67
|
+
env:
|
|
68
|
+
MYSQL_HOST: 127.0.0.1
|
|
69
|
+
MYSQL_PORT: 3306
|
|
70
|
+
MYSQL_USER: root
|
|
71
|
+
MYSQL_PASSWORD: root
|
|
72
|
+
MYSQL_DATABASE: e2e_source
|
|
73
|
+
ES_HOST: 127.0.0.1
|
|
74
|
+
ES_PORT: 9200
|
|
75
|
+
ES_MAJOR_VERSION: ${{ matrix.es_major }}
|
|
76
|
+
|
|
77
|
+
steps:
|
|
78
|
+
- uses: actions/checkout@v5
|
|
79
|
+
|
|
80
|
+
- name: Install MySQL client headers
|
|
81
|
+
run: sudo apt-get update && sudo apt-get install -y default-libmysqlclient-dev
|
|
82
|
+
|
|
83
|
+
- uses: ruby/setup-ruby@v1
|
|
84
|
+
with:
|
|
85
|
+
ruby-version: '3.4'
|
|
86
|
+
bundler-cache: true
|
|
87
|
+
|
|
88
|
+
- name: Wait for Elasticsearch
|
|
89
|
+
run: |
|
|
90
|
+
for i in $(seq 1 30); do
|
|
91
|
+
if curl -fs "http://127.0.0.1:9200/_cluster/health?wait_for_status=yellow&timeout=5s" >/dev/null; then
|
|
92
|
+
echo "Elasticsearch is up"; exit 0
|
|
93
|
+
fi
|
|
94
|
+
echo "waiting for Elasticsearch ($i)..."; sleep 5
|
|
95
|
+
done
|
|
96
|
+
echo "Elasticsearch did not become ready"; exit 1
|
|
97
|
+
|
|
98
|
+
- name: Run single-table replication E2E test
|
|
99
|
+
run: bundle exec ruby test/e2e/replication_single_e2e_test.rb
|
|
100
|
+
|
|
101
|
+
- name: Run multi-table replication E2E test
|
|
102
|
+
run: bundle exec ruby test/e2e/replication_multi_e2e_test.rb
|
|
103
|
+
|
|
104
|
+
# Container path: the official fluent/fluentd Docker image (gem-based).
|
|
105
|
+
install-smoke-docker:
|
|
106
|
+
name: Install smoke (Docker fluent/fluentd ${{ matrix.fluentd_tag }})
|
|
107
|
+
runs-on: ubuntu-latest
|
|
108
|
+
|
|
109
|
+
strategy:
|
|
110
|
+
fail-fast: false
|
|
111
|
+
matrix:
|
|
112
|
+
# "edge" tracks the latest Fluentd release to catch drift early. The
|
|
113
|
+
# packaged-distribution path is covered by install-smoke-fluent-package.
|
|
114
|
+
fluentd_tag: ['edge-debian']
|
|
115
|
+
|
|
116
|
+
steps:
|
|
117
|
+
- uses: actions/checkout@v5
|
|
118
|
+
|
|
119
|
+
- uses: ruby/setup-ruby@v1
|
|
120
|
+
with:
|
|
121
|
+
ruby-version: '3.4'
|
|
122
|
+
|
|
123
|
+
- name: Build gem
|
|
124
|
+
run: |
|
|
125
|
+
mkdir -p pkg
|
|
126
|
+
gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
|
|
127
|
+
|
|
128
|
+
- name: Install on official fluentd image (reproduces README steps)
|
|
129
|
+
run: |
|
|
130
|
+
docker run --rm --user root --entrypoint sh \
|
|
131
|
+
-v "${{ github.workspace }}:/workspace:ro" \
|
|
132
|
+
"fluent/fluentd:${{ matrix.fluentd_tag }}" \
|
|
133
|
+
/workspace/ci/install_smoke.sh
|
|
134
|
+
|
|
135
|
+
# Package path: fluent-package, the LTS distribution that replaced td-agent
|
|
136
|
+
# and is what most production users run today. v6 LTS bundles Fluentd v1.19.
|
|
137
|
+
install-smoke-fluent-package:
|
|
138
|
+
name: Install smoke (fluent-package v6 LTS)
|
|
139
|
+
runs-on: ubuntu-latest # GitHub's ubuntu-latest is Ubuntu 24.04 (Noble)
|
|
140
|
+
|
|
141
|
+
steps:
|
|
142
|
+
- uses: actions/checkout@v5
|
|
143
|
+
|
|
144
|
+
- uses: ruby/setup-ruby@v1
|
|
145
|
+
with:
|
|
146
|
+
ruby-version: '3.4'
|
|
147
|
+
|
|
148
|
+
- name: Build gem
|
|
149
|
+
run: |
|
|
150
|
+
mkdir -p pkg
|
|
151
|
+
gem build fluent-plugin-mysql-replicator.gemspec -o pkg/fluent-plugin-mysql-replicator.gem
|
|
152
|
+
|
|
153
|
+
- name: Install fluent-package v6 LTS
|
|
154
|
+
run: |
|
|
155
|
+
curl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh
|
|
156
|
+
/opt/fluent/bin/fluentd --version
|
|
157
|
+
|
|
158
|
+
- name: Install MySQL client headers
|
|
159
|
+
run: sudo apt-get update && sudo apt-get install -y build-essential default-libmysqlclient-dev
|
|
160
|
+
|
|
161
|
+
- name: Install plugin via fluent-gem and verify it loads
|
|
162
|
+
run: |
|
|
163
|
+
sudo /opt/fluent/bin/fluent-gem install pkg/fluent-plugin-mysql-replicator.gem
|
|
164
|
+
sudo /opt/fluent/bin/fluent-gem list fluent-plugin-mysql-replicator
|
|
165
|
+
/opt/fluent/bin/ruby ci/verify_plugins_load.rb
|
data/.gitignore
CHANGED
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [1.2.0] - 2026-06-16
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- JSON column support for nested object indexing. The new opt-in `json_columns`
|
|
12
|
+
option parses MySQL `JSON` string columns into nested objects before they are
|
|
13
|
+
emitted, so Elasticsearch indexes them as real nested JSON instead of escaped
|
|
14
|
+
strings. Available on both `mysql_replicator` (config option) and
|
|
15
|
+
`mysql_replicator_multi` (new `json_columns` column on the `settings` table).
|
|
16
|
+
Malformed JSON and non-string values are left untouched. ([#48], implements [#18])
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
- Delete detection no longer raises `bad value for range` (and no longer
|
|
20
|
+
allocates a huge array or emits phantom delete events) when the `primary_key`
|
|
21
|
+
is non-integer (e.g. a UUID `CHAR(36)`) or a large/sparse integer. The first
|
|
22
|
+
poll now only establishes a baseline. ([#42])
|
|
23
|
+
- In `mysql_replicator`, a nested sub-query is now triggered only when a column
|
|
24
|
+
value is an actual query template containing a `${placeholder}`. Previously any
|
|
25
|
+
value that merely began with the word `SELECT` was executed as SQL. This aligns
|
|
26
|
+
the single plugin with the fix already present in `mysql_replicator_multi`. ([#4])
|
|
27
|
+
|
|
28
|
+
### Documentation
|
|
29
|
+
- Document that the `mysql2` native extension requires the MySQL client
|
|
30
|
+
development headers (`default-libmysqlclient-dev` / `mysql-devel`), add a
|
|
31
|
+
"Failed to build gem native extension" troubleshooting note, and clarify that
|
|
32
|
+
change detection uses an in-memory hash of every row (not an `updated_at`
|
|
33
|
+
column). ([#43], [#40])
|
|
34
|
+
|
|
35
|
+
### CI
|
|
36
|
+
- Add install-smoke jobs that build and install the gem on **fluent-package v6
|
|
37
|
+
LTS** (the td-agent successor) and on the official `fluent/fluentd` image,
|
|
38
|
+
guarding against native-extension build regressions.
|
|
39
|
+
|
|
40
|
+
## [1.1.0] - 2026-06-15
|
|
41
|
+
|
|
42
|
+
### Added
|
|
43
|
+
- Auto-detect the Elasticsearch version on first write and support Elasticsearch
|
|
44
|
+
6.x through 9.x. The `_type` field is omitted automatically for 7.x and later,
|
|
45
|
+
where mapping types were removed.
|
|
46
|
+
- Dev Container configuration and end-to-end (E2E) replication tests, including
|
|
47
|
+
multi-table replication.
|
|
48
|
+
|
|
49
|
+
### Changed
|
|
50
|
+
- Migrated CI to GitHub Actions (removed Travis CI) and run the unit-test matrix
|
|
51
|
+
on Ruby 3.2–4.0, aligned with supported Fluentd versions.
|
|
52
|
+
|
|
53
|
+
## [1.0.3] - 2024-02-28
|
|
54
|
+
|
|
55
|
+
### Added
|
|
56
|
+
- Reconnection option to recover from database connection errors. ([#45])
|
|
57
|
+
|
|
58
|
+
## [1.0.2] - 2021-10-27
|
|
59
|
+
|
|
60
|
+
### Fixed
|
|
61
|
+
- Manage threads correctly in `mysql_replicator_multi`. ([#39])
|
|
62
|
+
|
|
63
|
+
## [1.0.1] - 2019-11-19
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
- Ensure the nested-query database connection (`nest_db`) is always closed.
|
|
67
|
+
|
|
68
|
+
### Documentation
|
|
69
|
+
- Documentation updates.
|
|
70
|
+
|
|
71
|
+
## [1.0.0] - 2019-11-13
|
|
72
|
+
|
|
73
|
+
- First 1.0 release, targeting the Fluentd v0.14+ plugin API. Earlier 0.x
|
|
74
|
+
history is available in the git log.
|
|
75
|
+
|
|
76
|
+
[1.2.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.1.0...v1.2.0
|
|
77
|
+
[1.1.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.3...v1.1.0
|
|
78
|
+
[1.0.3]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.2...v1.0.3
|
|
79
|
+
[1.0.2]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.1...v1.0.2
|
|
80
|
+
[1.0.1]: https://github.com/y-ken/fluent-plugin-mysql-replicator/compare/v1.0.0...v1.0.1
|
|
81
|
+
[1.0.0]: https://github.com/y-ken/fluent-plugin-mysql-replicator/releases/tag/v1.0.0
|
|
82
|
+
|
|
83
|
+
[#4]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/4
|
|
84
|
+
[#18]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/18
|
|
85
|
+
[#39]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/39
|
|
86
|
+
[#40]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/40
|
|
87
|
+
[#42]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/42
|
|
88
|
+
[#43]: https://github.com/y-ken/fluent-plugin-mysql-replicator/issues/43
|
|
89
|
+
[#45]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/45
|
|
90
|
+
[#48]: https://github.com/y-ken/fluent-plugin-mysql-replicator/pull/48
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# fluent-plugin-mysql-replicator [](https://github.com/y-ken/fluent-plugin-mysql-replicator/actions/workflows/ci.yml)
|
|
2
2
|
|
|
3
3
|
## Overview
|
|
4
4
|
|
|
@@ -15,14 +15,28 @@ It's comming support replicate to another RDB/noSQL.
|
|
|
15
15
|
|
|
16
16
|
## Dependency
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
This plugin depends on the [mysql2](https://rubygems.org/gems/mysql2) gem, which
|
|
19
|
+
builds a native extension at install time. You need both a C compiler toolchain
|
|
20
|
+
**and the MySQL client development headers** before installing.
|
|
19
21
|
|
|
20
22
|
```bash
|
|
21
|
-
# for RHEL/CentOS
|
|
23
|
+
# for RHEL/CentOS/Fedora
|
|
22
24
|
$ sudo yum group install "Development Tools"
|
|
25
|
+
$ sudo yum install mysql-devel # or mariadb-devel
|
|
23
26
|
|
|
24
|
-
# for Ubuntu/Debian
|
|
25
|
-
$ sudo apt-get install build-essential
|
|
27
|
+
# for Ubuntu/Debian (including the official fluent/fluentd Docker image)
|
|
28
|
+
$ sudo apt-get install build-essential default-libmysqlclient-dev
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### Troubleshooting: "Failed to build gem native extension"
|
|
32
|
+
|
|
33
|
+
If `gem install` (or `td-agent-gem install`) fails while building the `mysql2`
|
|
34
|
+
native extension, the MySQL client headers are almost always missing. Install
|
|
35
|
+
the development package for your platform shown above, then retry the
|
|
36
|
+
installation. On the official `fluent/fluentd:*-debian` images, run:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
$ apt-get update && apt-get install -y build-essential default-libmysqlclient-dev
|
|
26
40
|
```
|
|
27
41
|
|
|
28
42
|
## Installation
|
|
@@ -40,6 +54,24 @@ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
|
|
|
40
54
|
$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
|
|
41
55
|
`````
|
|
42
56
|
|
|
57
|
+
## Development container
|
|
58
|
+
|
|
59
|
+
This repository includes a VS Code Dev Container configuration under `.devcontainer/`.
|
|
60
|
+
Use Docker and Remote Containers / Dev Containers in VS Code to build and open the workspace inside a container.
|
|
61
|
+
The container installs Ruby, Bundler, and required native build dependencies.
|
|
62
|
+
|
|
63
|
+
After opening the repository in the Dev Container, run:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
bundle install --path vendor/bundle
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Then run tests like:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
bundle exec ruby -Itest test/plugin/test_out_mysql_replicator_elasticsearch.rb
|
|
73
|
+
```
|
|
74
|
+
|
|
43
75
|
## Included plugins
|
|
44
76
|
|
|
45
77
|
* Input Plugin: mysql_replicator
|
|
@@ -47,6 +79,49 @@ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
|
|
|
47
79
|
* Output Plugin: mysql_replicator_elasticsearch
|
|
48
80
|
* Output Plugin: mysql_replicator_solr (experimental)
|
|
49
81
|
|
|
82
|
+
## Elasticsearch version compatibility
|
|
83
|
+
|
|
84
|
+
`mysql_replicator_elasticsearch` works with Elasticsearch 6.x through 9.x.
|
|
85
|
+
|
|
86
|
+
Mapping types were removed in Elasticsearch 8.x (and deprecated in 7.x), so the
|
|
87
|
+
`_type` field can no longer be sent in bulk requests. The plugin detects the
|
|
88
|
+
Elasticsearch version on the first write and automatically omits `_type` for
|
|
89
|
+
7.x and later, so no extra configuration is required.
|
|
90
|
+
|
|
91
|
+
## JSON column support
|
|
92
|
+
|
|
93
|
+
MySQL `JSON` columns (MySQL 5.7.8+ / 8.x) are returned by the driver as plain
|
|
94
|
+
strings, so by default they reach Elasticsearch as escaped strings rather than
|
|
95
|
+
nested objects. List such columns in `json_columns` to have their values parsed
|
|
96
|
+
into nested objects before they are emitted.
|
|
97
|
+
|
|
98
|
+
* `mysql_replicator` (single): set the `json_columns` option (comma-separated).
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
<source>
|
|
102
|
+
@type mysql_replicator
|
|
103
|
+
# ...
|
|
104
|
+
query SELECT id, name, geometry FROM places
|
|
105
|
+
json_columns geometry,attrs
|
|
106
|
+
</source>
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
* `mysql_replicator_multi`: set the `json_columns` column (comma-separated) on
|
|
110
|
+
the relevant row of the `settings` management table. Existing installs can add
|
|
111
|
+
the column with:
|
|
112
|
+
|
|
113
|
+
```sql
|
|
114
|
+
ALTER TABLE settings ADD COLUMN `json_columns` varchar(255) DEFAULT NULL AFTER `primary_key`;
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Notes:
|
|
118
|
+
|
|
119
|
+
* This option is intended for Elasticsearch. **Do not set it when the destination
|
|
120
|
+
cannot store JSON objects (e.g. the Solr output)** — leave it empty there.
|
|
121
|
+
* Only top-level columns are parsed (columns inside nested documents are not).
|
|
122
|
+
* Malformed JSON and non-string values are left untouched, so enabling the option
|
|
123
|
+
never corrupts non-JSON data.
|
|
124
|
+
|
|
50
125
|
## Output example
|
|
51
126
|
|
|
52
127
|
It is a example when detecting insert/update/delete events.
|
|
@@ -8,6 +8,26 @@ It is useful for these purpose.
|
|
|
8
8
|
**Note:**
|
|
9
9
|
On syncing 300 million rows table, it will consume around 800MB of memory with ruby 1.9.3 environment.
|
|
10
10
|
|
|
11
|
+
### How it works
|
|
12
|
+
|
|
13
|
+
`mysql_replicator` does **not** rely on any `updated_at`/timestamp column to
|
|
14
|
+
detect changes. On every `interval`, it re-runs the configured `query`, computes
|
|
15
|
+
a hash for each returned row, and keeps an **in-memory hash table of every row**
|
|
16
|
+
(keyed by `primary_key`) to compare against the previous run:
|
|
17
|
+
|
|
18
|
+
* a row whose `primary_key` was not seen before → `insert` event
|
|
19
|
+
* a row whose hash changed since the previous run → `update` event
|
|
20
|
+
* a `primary_key` that disappeared from the result set → `delete` event (when `enable_delete yes`)
|
|
21
|
+
|
|
22
|
+
Because the whole result set is held in memory, this plugin is best suited for
|
|
23
|
+
small-to-medium tables. For millions of rows or multiple tables, prefer
|
|
24
|
+
[`mysql_replicator_multi`](Tutorial-mysql_replicator_multi.md), which stores the
|
|
25
|
+
hash table in a MySQL management table instead of Ruby memory.
|
|
26
|
+
|
|
27
|
+
The `updated_at` column is therefore not required. It only becomes useful if you
|
|
28
|
+
intentionally narrow the `query` to recently changed rows together with
|
|
29
|
+
`enable_delete no`, as shown in the `enable_delete` comment below.
|
|
30
|
+
|
|
11
31
|
### configuration
|
|
12
32
|
|
|
13
33
|
`````
|
|
@@ -21,7 +41,7 @@ On syncing 300 million rows table, it will consume around 800MB of memory with r
|
|
|
21
41
|
database myweb
|
|
22
42
|
|
|
23
43
|
# Set replicate query configuration.
|
|
24
|
-
query SELECT id, text
|
|
44
|
+
query SELECT id, text from search_test;
|
|
25
45
|
primary_key id # specify unique key (default: id)
|
|
26
46
|
interval 10s # execute query interval (default: 1m)
|
|
27
47
|
|
data/ci/install_smoke.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Install smoke test: reproduces the README install steps inside the official
|
|
3
|
+
# fluent/fluentd Debian image to verify the mysql2 native extension actually
|
|
4
|
+
# builds and the plugins load. Guards against the "Failed to build gem native
|
|
5
|
+
# extension" class of breakage reported in #43 / #16 / #35.
|
|
6
|
+
set -eux
|
|
7
|
+
|
|
8
|
+
GEM_FILE="${GEM_FILE:-/workspace/pkg/fluent-plugin-mysql-replicator.gem}"
|
|
9
|
+
|
|
10
|
+
# Build toolchain + MySQL client development headers (the bit users forget).
|
|
11
|
+
apt-get update
|
|
12
|
+
apt-get install -y build-essential default-libmysqlclient-dev
|
|
13
|
+
|
|
14
|
+
# Installing the gem compiles the mysql2 native extension; this fails loudly
|
|
15
|
+
# if the client headers are missing, which is exactly what we want to catch.
|
|
16
|
+
gem install "$GEM_FILE"
|
|
17
|
+
|
|
18
|
+
# Confirm the installed plugins (and the mysql2 native extension) load at runtime.
|
|
19
|
+
ruby "$(dirname "$0")/verify_plugins_load.rb"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# Loads the plugins to confirm they (and the mysql2 native extension) are
|
|
2
|
+
# usable at runtime after a fresh install. Shared by both install-smoke jobs.
|
|
3
|
+
require "fluent/plugin/in_mysql_replicator"
|
|
4
|
+
require "fluent/plugin/in_mysql_replicator_multi"
|
|
5
|
+
puts "input plugins loaded; mysql2 native extension OK"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
|
2
2
|
Gem::Specification.new do |s|
|
|
3
3
|
s.name = "fluent-plugin-mysql-replicator"
|
|
4
|
-
s.version = "1.0
|
|
4
|
+
s.version = "1.2.0"
|
|
5
5
|
s.authors = ["Kentaro Yoshida"]
|
|
6
6
|
s.email = ["y.ken.studio@gmail.com"]
|
|
7
7
|
s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'mysql2'
|
|
2
2
|
require 'digest/sha1'
|
|
3
|
+
require 'json'
|
|
3
4
|
require 'fluent/plugin/input'
|
|
4
5
|
|
|
5
6
|
module Fluent::Plugin
|
|
@@ -19,6 +20,10 @@ module Fluent::Plugin
|
|
|
19
20
|
config_param :primary_key, :string, :default => 'id'
|
|
20
21
|
config_param :interval, :string, :default => '1m'
|
|
21
22
|
config_param :enable_delete, :bool, :default => true
|
|
23
|
+
# Comma-separated column names whose MySQL JSON values should be parsed into
|
|
24
|
+
# nested objects before emitting. Intended for Elasticsearch; do not set it
|
|
25
|
+
# when the destination cannot store JSON objects (e.g. Solr).
|
|
26
|
+
config_param :json_columns, :array, :default => []
|
|
22
27
|
config_param :tag, :string, :default => nil
|
|
23
28
|
|
|
24
29
|
def configure(conf)
|
|
@@ -29,7 +34,7 @@ module Fluent::Plugin
|
|
|
29
34
|
raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
|
|
30
35
|
end
|
|
31
36
|
|
|
32
|
-
log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
|
|
37
|
+
log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete} :json_columns=>#{@json_columns}"
|
|
33
38
|
end
|
|
34
39
|
|
|
35
40
|
def start
|
|
@@ -71,7 +76,8 @@ module Fluent::Plugin
|
|
|
71
76
|
current_ids << row[@primary_key]
|
|
72
77
|
current_hash = Digest::SHA1.hexdigest(row.flatten.join)
|
|
73
78
|
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
|
74
|
-
row
|
|
79
|
+
parse_json_columns!(row, @json_columns)
|
|
80
|
+
row.select {|k, v| nested_query_value?(v) }.each do |k, v|
|
|
75
81
|
row[k] = [] unless row[k].is_a?(Array)
|
|
76
82
|
nest_rows, prepared_con = query(v.gsub(/\$\{([^\}]+)\}/, row[$1].to_s), prepared_con)
|
|
77
83
|
nest_rows.each do |nest_row|
|
|
@@ -97,13 +103,7 @@ module Fluent::Plugin
|
|
|
97
103
|
con.close
|
|
98
104
|
ids = current_ids
|
|
99
105
|
if @enable_delete
|
|
100
|
-
|
|
101
|
-
deleted_ids = Array.new
|
|
102
|
-
elsif previous_ids.empty?
|
|
103
|
-
deleted_ids = [*1...current_ids.max] - current_ids
|
|
104
|
-
else
|
|
105
|
-
deleted_ids = previous_ids - current_ids
|
|
106
|
-
end
|
|
106
|
+
deleted_ids = detect_deleted_ids(previous_ids, current_ids)
|
|
107
107
|
if deleted_ids.count > 0
|
|
108
108
|
hash_delete_by_list(table_hash, deleted_ids)
|
|
109
109
|
deleted_ids.each do |id|
|
|
@@ -122,6 +122,35 @@ module Fluent::Plugin
|
|
|
122
122
|
deleted_keys.each{|k| hash.delete(k)}
|
|
123
123
|
end
|
|
124
124
|
|
|
125
|
+
# Returns the primary keys that disappeared since the previous poll.
|
|
126
|
+
#
|
|
127
|
+
# The first poll only establishes a baseline: there is no previous snapshot
|
|
128
|
+
# to diff against, so nothing is reported as deleted yet. This also avoids
|
|
129
|
+
# the old `[*1...current_ids.max]` range, which raised "bad value for range"
|
|
130
|
+
# for non-integer primary keys and allocated a huge array (and emitted
|
|
131
|
+
# phantom deletes) for large / sparse integer ids. (#42)
|
|
132
|
+
def detect_deleted_ids(previous_ids, current_ids)
|
|
133
|
+
return [] if previous_ids.empty?
|
|
134
|
+
return [] if current_ids.empty?
|
|
135
|
+
previous_ids - current_ids
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Parse the given columns' JSON string values into nested objects in place.
|
|
139
|
+
# Non-string values, missing columns, and malformed JSON are left untouched
|
|
140
|
+
# so enabling this never corrupts non-JSON data.
|
|
141
|
+
def parse_json_columns!(row, columns)
|
|
142
|
+
return if columns.empty?
|
|
143
|
+
columns.each do |col|
|
|
144
|
+
v = row[col]
|
|
145
|
+
next unless v.is_a?(String)
|
|
146
|
+
begin
|
|
147
|
+
row[col] = JSON.parse(v)
|
|
148
|
+
rescue JSON::ParserError
|
|
149
|
+
# leave the original string as-is on malformed JSON
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
125
154
|
def format_tag(tag, param)
|
|
126
155
|
pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
|
|
127
156
|
tag.gsub(/(\${[a-z_]+})/) do
|
|
@@ -130,6 +159,15 @@ module Fluent::Plugin
|
|
|
130
159
|
end
|
|
131
160
|
end
|
|
132
161
|
|
|
162
|
+
# A column value triggers a nested sub-query only when it is a query
|
|
163
|
+
# template containing a `${placeholder}` (e.g. "SELECT ... WHERE x = ${id}").
|
|
164
|
+
# Requiring the placeholder prevents ordinary text values that merely begin
|
|
165
|
+
# with the word "SELECT" from being executed as SQL. (#4; mirrors the fix
|
|
166
|
+
# already applied to mysql_replicator_multi in #6.)
|
|
167
|
+
def nested_query_value?(value)
|
|
168
|
+
value.to_s.strip.match?(/^SELECT[^\$]+\$\{[^\}]+\}/i)
|
|
169
|
+
end
|
|
170
|
+
|
|
133
171
|
def emit_record(tag, record)
|
|
134
172
|
router.emit(tag, Fluent::Engine.now, record)
|
|
135
173
|
end
|
|
@@ -9,6 +9,7 @@ module Fluent::Plugin
|
|
|
9
9
|
def initialize
|
|
10
10
|
require 'mysql2'
|
|
11
11
|
require 'digest/sha1'
|
|
12
|
+
require 'json'
|
|
12
13
|
super
|
|
13
14
|
end
|
|
14
15
|
|
|
@@ -81,6 +82,7 @@ module Fluent::Plugin
|
|
|
81
82
|
log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
|
|
82
83
|
}
|
|
83
84
|
primary_key = config['primary_key']
|
|
85
|
+
json_columns = (config['json_columns'] || '').split(',').map(&:strip).reject(&:empty?)
|
|
84
86
|
previous_id = current_id = nil
|
|
85
87
|
while @running
|
|
86
88
|
rows_count = 0
|
|
@@ -97,6 +99,7 @@ module Fluent::Plugin
|
|
|
97
99
|
db = get_origin_connection(config)
|
|
98
100
|
db.query(config['query']).each do |row|
|
|
99
101
|
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
|
102
|
+
parse_json_columns!(row, json_columns)
|
|
100
103
|
row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
|
|
101
104
|
row[k] = [] unless row[k].is_a?(Array)
|
|
102
105
|
nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
|
|
@@ -266,6 +269,22 @@ module Fluent::Plugin
|
|
|
266
269
|
@hash_table_bulk_insert_last_time = Time.now
|
|
267
270
|
end
|
|
268
271
|
|
|
272
|
+
# Parse the given columns' JSON string values into nested objects in place.
|
|
273
|
+
# Non-string values, missing columns, and malformed JSON are left untouched
|
|
274
|
+
# so enabling this never corrupts non-JSON data.
|
|
275
|
+
def parse_json_columns!(row, columns)
|
|
276
|
+
return if columns.empty?
|
|
277
|
+
columns.each do |col|
|
|
278
|
+
v = row[col]
|
|
279
|
+
next unless v.is_a?(String)
|
|
280
|
+
begin
|
|
281
|
+
row[col] = JSON.parse(v)
|
|
282
|
+
rescue JSON::ParserError
|
|
283
|
+
# leave the original string as-is on malformed JSON
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
269
288
|
def emit_record(tag, record)
|
|
270
289
|
router.emit(tag, Fluent::Engine.now, record)
|
|
271
290
|
end
|