fluent-plugin-mysql-replicator 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +2 -2
- data/fluent-plugin-mysql-replicator.gemspec +1 -1
- data/lib/fluent/plugin/in_mysql_replicator_multi.rb +59 -31
- data/setup_mysql_replicator_multi.sql +4 -0
- metadata +9 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 76bd65971409a89e44f0cc67010a9365b1d78bd195d68bb1ddffd90294b6059c
|
4
|
+
data.tar.gz: 27f0bedbb9814a9e08d36bdc53bde79c142465cbb928629695464ebfe25150af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b6eb13f3bb9da56d5de5b474388c40582de0cb06412eb1d83c8bd635f979dc88c4464ff8e570998f16d458db9746d1e0eb1a312e1648215474b2991a622e7684
|
7
|
+
data.tar.gz: a1bd184c16ca5d2a730884a52db60977510302f40d11b7f41c52e2e6cbaa5a93e8e208d0c1371276bd5b7cee839561e8676b4cb63f7657133695c2d667374897
|
data/README.md
CHANGED
@@ -31,13 +31,13 @@ install with gem or fluent-gem command as:
|
|
31
31
|
|
32
32
|
`````
|
33
33
|
# for system installed fluentd
|
34
|
-
$ gem install fluent-plugin-mysql-replicator -v 1.0.
|
34
|
+
$ gem install fluent-plugin-mysql-replicator -v 1.0.3
|
35
35
|
|
36
36
|
# for td-agent2
|
37
37
|
$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
|
38
38
|
|
39
39
|
# for td-agent3
|
40
|
-
$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.
|
40
|
+
$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
|
41
41
|
`````
|
42
42
|
|
43
43
|
## Included plugins
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |s|
|
3
3
|
s.name = "fluent-plugin-mysql-replicator"
|
4
|
-
s.version = "1.0.
|
4
|
+
s.version = "1.0.3"
|
5
5
|
s.authors = ["Kentaro Yoshida"]
|
6
6
|
s.email = ["y.ken.studio@gmail.com"]
|
7
7
|
s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
|
@@ -1,9 +1,11 @@
|
|
1
|
-
require 'fluent/input'
|
1
|
+
require 'fluent/plugin/input'
|
2
2
|
|
3
3
|
module Fluent::Plugin
|
4
|
-
class MysqlReplicatorMultiInput < Fluent::Input
|
4
|
+
class MysqlReplicatorMultiInput < Fluent::Plugin::Input
|
5
5
|
Fluent::Plugin.register_input('mysql_replicator_multi', self)
|
6
6
|
|
7
|
+
helpers :thread
|
8
|
+
|
7
9
|
def initialize
|
8
10
|
require 'mysql2'
|
9
11
|
require 'digest/sha1'
|
@@ -34,6 +36,7 @@ module Fluent::Plugin
|
|
34
36
|
@mutex = Mutex.new
|
35
37
|
@manager_db = get_manager_connection
|
36
38
|
@manager_db.query("SET SESSION wait_timeout=1800;")
|
39
|
+
@running = true
|
37
40
|
@threads << thread_create(:in_mysql_replicator_flusher) {
|
38
41
|
@hash_table_bulk_insert = []
|
39
42
|
@hash_table_bulk_insert_last_time = Time.now
|
@@ -51,7 +54,13 @@ module Fluent::Plugin
|
|
51
54
|
end
|
52
55
|
end
|
53
56
|
|
57
|
+
def stop
|
58
|
+
@running = false
|
59
|
+
super
|
60
|
+
end
|
61
|
+
|
54
62
|
def shutdown
|
63
|
+
@threads.each(&:join)
|
55
64
|
super
|
56
65
|
end
|
57
66
|
|
@@ -73,40 +82,52 @@ module Fluent::Plugin
|
|
73
82
|
}
|
74
83
|
primary_key = config['primary_key']
|
75
84
|
previous_id = current_id = nil
|
76
|
-
|
85
|
+
while @running
|
77
86
|
rows_count = 0
|
78
87
|
start_time = Time.now
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
88
|
+
db = nil
|
89
|
+
nest_db = nil
|
90
|
+
begin
|
91
|
+
unless config['prepared_query'].nil?
|
92
|
+
nest_db = get_origin_connection(config)
|
93
|
+
config['prepared_query'].strip.split(/;/).each do |query|
|
94
|
+
nest_db.query(query)
|
95
|
+
end
|
83
96
|
end
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
97
|
+
db = get_origin_connection(config)
|
98
|
+
db.query(config['query']).each do |row|
|
99
|
+
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
100
|
+
row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
|
101
|
+
row[k] = [] unless row[k].is_a?(Array)
|
102
|
+
nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
|
103
|
+
nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
104
|
+
row[k] << nest_row
|
105
|
+
end
|
93
106
|
end
|
107
|
+
current_id = row[primary_key]
|
108
|
+
@mutex.synchronize {
|
109
|
+
if row[primary_key].nil?
|
110
|
+
log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
|
111
|
+
break
|
112
|
+
end
|
113
|
+
detect_insert_update(config, row)
|
114
|
+
detect_delete(config, current_id, previous_id)
|
115
|
+
}
|
116
|
+
previous_id = current_id
|
117
|
+
rows_count += 1
|
94
118
|
end
|
95
|
-
|
119
|
+
rescue Mysql2::Error => e
|
120
|
+
raise e unless config['enable_retry'] == 1
|
121
|
+
|
96
122
|
@mutex.synchronize {
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
end
|
101
|
-
detect_insert_update(config, row)
|
102
|
-
detect_delete(config, current_id, previous_id)
|
123
|
+
log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
|
124
|
+
log.error "error: #{e.message}"
|
125
|
+
log.error e.backtrace.join("\n")
|
103
126
|
}
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
unless config['prepared_query'].nil?
|
109
|
-
nest_db.close
|
127
|
+
sleep config['retry_interval']
|
128
|
+
ensure
|
129
|
+
db.close if db
|
130
|
+
nest_db.close if nest_db && !config['prepared_query'].nil?
|
110
131
|
end
|
111
132
|
elapsed_time = sprintf("%0.02f", Time.now - start_time)
|
112
133
|
@mutex.synchronize {
|
@@ -214,7 +235,7 @@ module Fluent::Plugin
|
|
214
235
|
|
215
236
|
def hash_table_flusher
|
216
237
|
begin
|
217
|
-
|
238
|
+
while @running
|
218
239
|
if @hash_table_bulk_insert.empty? || @bulk_insert_timeout > (Time.now - @hash_table_bulk_insert_last_time)
|
219
240
|
sleep @bulk_insert_timeout
|
220
241
|
next
|
@@ -223,6 +244,9 @@ module Fluent::Plugin
|
|
223
244
|
flush_hash_table
|
224
245
|
}
|
225
246
|
end
|
247
|
+
@mutex.synchronize {
|
248
|
+
flush_hash_table
|
249
|
+
}
|
226
250
|
rescue StandardError => e
|
227
251
|
@mutex.synchronize {
|
228
252
|
log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
|
@@ -278,7 +302,11 @@ module Fluent::Plugin
|
|
278
302
|
:cache_rows => false
|
279
303
|
)
|
280
304
|
rescue Mysql2::Error => e
|
281
|
-
|
305
|
+
if config['enable_retry'] == 1
|
306
|
+
raise e
|
307
|
+
else
|
308
|
+
raise "mysql_replicator_multi: #{e}"
|
309
|
+
end
|
282
310
|
end
|
283
311
|
end
|
284
312
|
end
|
@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
|
|
29
29
|
`enable_loose_insert` int(11) DEFAULT '0',
|
30
30
|
-- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
|
31
31
|
`enable_loose_delete` int(11) DEFAULT '0',
|
32
|
+
-- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
|
33
|
+
`enable_retry` int(11) DEFAULT '1',
|
34
|
+
-- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
|
35
|
+
`retry_interval` int(11) NOT NULL DEFAULT '30',
|
32
36
|
PRIMARY KEY (`id`),
|
33
37
|
UNIQUE KEY `name` (`name`)
|
34
38
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-replicator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -149,11 +149,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
149
|
- !ruby/object:Gem::Version
|
150
150
|
version: '0'
|
151
151
|
requirements: []
|
152
|
-
|
153
|
-
rubygems_version: 2.6.11
|
152
|
+
rubygems_version: 3.1.6
|
154
153
|
signing_key:
|
155
154
|
specification_version: 4
|
156
155
|
summary: Fluentd input plugin to track insert/update/delete event from MySQL database
|
157
156
|
server. Not only that, it could multiple table replication and generate nested document
|
158
157
|
for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.
|
159
|
-
test_files:
|
158
|
+
test_files:
|
159
|
+
- test/helper.rb
|
160
|
+
- test/plugin/test_in_mysql_replicator.rb
|
161
|
+
- test/plugin/test_in_mysql_replicator_multi.rb
|
162
|
+
- test/plugin/test_out_mysql_replicator_elasticsearch.rb
|
163
|
+
- test/plugin/test_out_mysql_replicator_solr.rb
|