fluent-plugin-mysql-replicator 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: f457b520cc136f8ec0b3c78ee690fa20e4ce65a9
4
- data.tar.gz: 2df7a1c2c1bbf24741b6fbf9ed35b819d499eb80
2
+ SHA256:
3
+ metadata.gz: 76bd65971409a89e44f0cc67010a9365b1d78bd195d68bb1ddffd90294b6059c
4
+ data.tar.gz: 27f0bedbb9814a9e08d36bdc53bde79c142465cbb928629695464ebfe25150af
5
5
  SHA512:
6
- metadata.gz: b96f9134fa44ca80468a77abbbc10f30f28f8a1c12cd60a91e56f0c1a4a9e8e4768d41faa67646f57ea3b045815eb753bf6e1a89bb92c5385faf2afdc6786f11
7
- data.tar.gz: '088c4f52b9565348fde62f86232b23f315b268db9e5d52bdcda7055eca2167b837336e0b18ede444657c0c0acf1c2083de245dd15c15f18e9caa529f44ebcf70'
6
+ metadata.gz: b6eb13f3bb9da56d5de5b474388c40582de0cb06412eb1d83c8bd635f979dc88c4464ff8e570998f16d458db9746d1e0eb1a312e1648215474b2991a622e7684
7
+ data.tar.gz: a1bd184c16ca5d2a730884a52db60977510302f40d11b7f41c52e2e6cbaa5a93e8e208d0c1371276bd5b7cee839561e8676b4cb63f7657133695c2d667374897
data/README.md CHANGED
@@ -31,13 +31,13 @@ install with gem or fluent-gem command as:
31
31
 
32
32
  `````
33
33
  # for system installed fluentd
34
- $ gem install fluent-plugin-mysql-replicator -v 1.0.1
34
+ $ gem install fluent-plugin-mysql-replicator -v 1.0.3
35
35
 
36
36
  # for td-agent2
37
37
  $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
38
38
 
39
39
  # for td-agent3
40
- $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.1
40
+ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
41
41
  `````
42
42
 
43
43
  ## Included plugins
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |s|
3
3
  s.name = "fluent-plugin-mysql-replicator"
4
- s.version = "1.0.1"
4
+ s.version = "1.0.3"
5
5
  s.authors = ["Kentaro Yoshida"]
6
6
  s.email = ["y.ken.studio@gmail.com"]
7
7
  s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
@@ -1,9 +1,11 @@
1
- require 'fluent/input'
1
+ require 'fluent/plugin/input'
2
2
 
3
3
  module Fluent::Plugin
4
- class MysqlReplicatorMultiInput < Fluent::Input
4
+ class MysqlReplicatorMultiInput < Fluent::Plugin::Input
5
5
  Fluent::Plugin.register_input('mysql_replicator_multi', self)
6
6
 
7
+ helpers :thread
8
+
7
9
  def initialize
8
10
  require 'mysql2'
9
11
  require 'digest/sha1'
@@ -34,6 +36,7 @@ module Fluent::Plugin
34
36
  @mutex = Mutex.new
35
37
  @manager_db = get_manager_connection
36
38
  @manager_db.query("SET SESSION wait_timeout=1800;")
39
+ @running = true
37
40
  @threads << thread_create(:in_mysql_replicator_flusher) {
38
41
  @hash_table_bulk_insert = []
39
42
  @hash_table_bulk_insert_last_time = Time.now
@@ -51,7 +54,13 @@ module Fluent::Plugin
51
54
  end
52
55
  end
53
56
 
57
+ def stop
58
+ @running = false
59
+ super
60
+ end
61
+
54
62
  def shutdown
63
+ @threads.each(&:join)
55
64
  super
56
65
  end
57
66
 
@@ -73,40 +82,52 @@ module Fluent::Plugin
73
82
  }
74
83
  primary_key = config['primary_key']
75
84
  previous_id = current_id = nil
76
- loop do
85
+ while @running
77
86
  rows_count = 0
78
87
  start_time = Time.now
79
- unless config['prepared_query'].nil?
80
- nest_db = get_origin_connection(config)
81
- config['prepared_query'].strip.split(/;/).each do |query|
82
- nest_db.query(query)
88
+ db = nil
89
+ nest_db = nil
90
+ begin
91
+ unless config['prepared_query'].nil?
92
+ nest_db = get_origin_connection(config)
93
+ config['prepared_query'].strip.split(/;/).each do |query|
94
+ nest_db.query(query)
95
+ end
83
96
  end
84
- end
85
- db = get_origin_connection(config)
86
- db.query(config['query']).each do |row|
87
- row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
88
- row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
89
- row[k] = [] unless row[k].is_a?(Array)
90
- nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
91
- nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
92
- row[k] << nest_row
97
+ db = get_origin_connection(config)
98
+ db.query(config['query']).each do |row|
99
+ row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
100
+ row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
101
+ row[k] = [] unless row[k].is_a?(Array)
102
+ nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
103
+ nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
104
+ row[k] << nest_row
105
+ end
93
106
  end
107
+ current_id = row[primary_key]
108
+ @mutex.synchronize {
109
+ if row[primary_key].nil?
110
+ log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
111
+ break
112
+ end
113
+ detect_insert_update(config, row)
114
+ detect_delete(config, current_id, previous_id)
115
+ }
116
+ previous_id = current_id
117
+ rows_count += 1
94
118
  end
95
- current_id = row[primary_key]
119
+ rescue Mysql2::Error => e
120
+ raise e unless config['enable_retry'] == 1
121
+
96
122
  @mutex.synchronize {
97
- if row[primary_key].nil?
98
- log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
99
- break
100
- end
101
- detect_insert_update(config, row)
102
- detect_delete(config, current_id, previous_id)
123
+ log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
124
+ log.error "error: #{e.message}"
125
+ log.error e.backtrace.join("\n")
103
126
  }
104
- previous_id = current_id
105
- rows_count += 1
106
- end
107
- db.close
108
- unless config['prepared_query'].nil?
109
- nest_db.close
127
+ sleep config['retry_interval']
128
+ ensure
129
+ db.close if db
130
+ nest_db.close if nest_db && !config['prepared_query'].nil?
110
131
  end
111
132
  elapsed_time = sprintf("%0.02f", Time.now - start_time)
112
133
  @mutex.synchronize {
@@ -214,7 +235,7 @@ module Fluent::Plugin
214
235
 
215
236
  def hash_table_flusher
216
237
  begin
217
- loop do
238
+ while @running
218
239
  if @hash_table_bulk_insert.empty? || @bulk_insert_timeout > (Time.now - @hash_table_bulk_insert_last_time)
219
240
  sleep @bulk_insert_timeout
220
241
  next
@@ -223,6 +244,9 @@ module Fluent::Plugin
223
244
  flush_hash_table
224
245
  }
225
246
  end
247
+ @mutex.synchronize {
248
+ flush_hash_table
249
+ }
226
250
  rescue StandardError => e
227
251
  @mutex.synchronize {
228
252
  log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
@@ -278,7 +302,11 @@ module Fluent::Plugin
278
302
  :cache_rows => false
279
303
  )
280
304
  rescue Mysql2::Error => e
281
- raise "mysql_replicator_multi: #{e}"
305
+ if config['enable_retry'] == 1
306
+ raise e
307
+ else
308
+ raise "mysql_replicator_multi: #{e}"
309
+ end
282
310
  end
283
311
  end
284
312
  end
@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
29
29
  `enable_loose_insert` int(11) DEFAULT '0',
30
30
  -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
31
31
  `enable_loose_delete` int(11) DEFAULT '0',
32
+ -- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
33
+ `enable_retry` int(11) DEFAULT '1',
34
+ -- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
35
+ `retry_interval` int(11) NOT NULL DEFAULT '30',
32
36
  PRIMARY KEY (`id`),
33
37
  UNIQUE KEY `name` (`name`)
34
38
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql-replicator
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Yoshida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-19 00:00:00.000000000 Z
11
+ date: 2024-03-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -149,11 +149,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
149
  - !ruby/object:Gem::Version
150
150
  version: '0'
151
151
  requirements: []
152
- rubyforge_project:
153
- rubygems_version: 2.6.11
152
+ rubygems_version: 3.1.6
154
153
  signing_key:
155
154
  specification_version: 4
156
155
  summary: Fluentd input plugin to track insert/update/delete event from MySQL database
157
156
  server. Not only that, it could multiple table replication and generate nested document
158
157
  for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.
159
- test_files: []
158
+ test_files:
159
+ - test/helper.rb
160
+ - test/plugin/test_in_mysql_replicator.rb
161
+ - test/plugin/test_in_mysql_replicator_multi.rb
162
+ - test/plugin/test_out_mysql_replicator_elasticsearch.rb
163
+ - test/plugin/test_out_mysql_replicator_solr.rb