fluent-plugin-mysql-replicator 1.0.2 → 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/fluent-plugin-mysql-replicator.gemspec +1 -1
  4. data/lib/fluent/plugin/in_mysql_replicator_multi.rb +43 -27
  5. data/setup_mysql_replicator_multi.sql +4 -0
  6. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5c523ddf1d5052d11c6c7a0f7e9172d319c881257d791a0006cd1a6601f5a948
4
- data.tar.gz: ca9f218dfa56543f141d8911578250194281bfbad16d8f5ed6b4980120d99c8d
3
+ metadata.gz: 76bd65971409a89e44f0cc67010a9365b1d78bd195d68bb1ddffd90294b6059c
4
+ data.tar.gz: 27f0bedbb9814a9e08d36bdc53bde79c142465cbb928629695464ebfe25150af
5
5
  SHA512:
6
- metadata.gz: a5266cd4f824e781d53eaa613c37f260c7814badcee87fb6c49a1321e9c9ac8bc8d9505b2ea800f3a8ccde237b2b0eeb808e187463996c2b1aecc93065128869
7
- data.tar.gz: 70834ef8629d34bbd53aa1ff9d66ba706d6df5b48cd310d8a554c7d25628682633860123f63855e851482abda385e6288c4fcd01da848cbffc56bc0525553968
6
+ metadata.gz: b6eb13f3bb9da56d5de5b474388c40582de0cb06412eb1d83c8bd635f979dc88c4464ff8e570998f16d458db9746d1e0eb1a312e1648215474b2991a622e7684
7
+ data.tar.gz: a1bd184c16ca5d2a730884a52db60977510302f40d11b7f41c52e2e6cbaa5a93e8e208d0c1371276bd5b7cee839561e8676b4cb63f7657133695c2d667374897
data/README.md CHANGED
@@ -31,13 +31,13 @@ install with gem or fluent-gem command as:
31
31
 
32
32
  `````
33
33
  # for system installed fluentd
34
- $ gem install fluent-plugin-mysql-replicator -v 1.0.2
34
+ $ gem install fluent-plugin-mysql-replicator -v 1.0.3
35
35
 
36
36
  # for td-agent2
37
37
  $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
38
38
 
39
39
  # for td-agent3
40
- $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.2
40
+ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
41
41
  `````
42
42
 
43
43
  ## Included plugins
data/fluent-plugin-mysql-replicator.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |s|
3
3
  s.name = "fluent-plugin-mysql-replicator"
4
- s.version = "1.0.2"
4
+ s.version = "1.0.3"
5
5
  s.authors = ["Kentaro Yoshida"]
6
6
  s.email = ["y.ken.studio@gmail.com"]
7
7
  s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
data/lib/fluent/plugin/in_mysql_replicator_multi.rb CHANGED
@@ -85,37 +85,49 @@ module Fluent::Plugin
85
85
  while @running
86
86
  rows_count = 0
87
87
  start_time = Time.now
88
- unless config['prepared_query'].nil?
89
- nest_db = get_origin_connection(config)
90
- config['prepared_query'].strip.split(/;/).each do |query|
91
- nest_db.query(query)
88
+ db = nil
89
+ nest_db = nil
90
+ begin
91
+ unless config['prepared_query'].nil?
92
+ nest_db = get_origin_connection(config)
93
+ config['prepared_query'].strip.split(/;/).each do |query|
94
+ nest_db.query(query)
95
+ end
92
96
  end
93
- end
94
- db = get_origin_connection(config)
95
- db.query(config['query']).each do |row|
96
- row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
97
- row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
98
- row[k] = [] unless row[k].is_a?(Array)
99
- nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
100
- nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
101
- row[k] << nest_row
97
+ db = get_origin_connection(config)
98
+ db.query(config['query']).each do |row|
99
+ row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
100
+ row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
101
+ row[k] = [] unless row[k].is_a?(Array)
102
+ nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
103
+ nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
104
+ row[k] << nest_row
105
+ end
102
106
  end
107
+ current_id = row[primary_key]
108
+ @mutex.synchronize {
109
+ if row[primary_key].nil?
110
+ log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
111
+ break
112
+ end
113
+ detect_insert_update(config, row)
114
+ detect_delete(config, current_id, previous_id)
115
+ }
116
+ previous_id = current_id
117
+ rows_count += 1
103
118
  end
104
- current_id = row[primary_key]
119
+ rescue Mysql2::Error => e
120
+ raise e unless config['enable_retry'] == 1
121
+
105
122
  @mutex.synchronize {
106
- if row[primary_key].nil?
107
- log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
108
- break
109
- end
110
- detect_insert_update(config, row)
111
- detect_delete(config, current_id, previous_id)
123
+ log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
124
+ log.error "error: #{e.message}"
125
+ log.error e.backtrace.join("\n")
112
126
  }
113
- previous_id = current_id
114
- rows_count += 1
115
- end
116
- db.close
117
- unless config['prepared_query'].nil?
118
- nest_db.close
127
+ sleep config['retry_interval']
128
+ ensure
129
+ db.close if db
130
+ nest_db.close if nest_db && !config['prepared_query'].nil?
119
131
  end
120
132
  elapsed_time = sprintf("%0.02f", Time.now - start_time)
121
133
  @mutex.synchronize {
@@ -290,7 +302,11 @@ module Fluent::Plugin
290
302
  :cache_rows => false
291
303
  )
292
304
  rescue Mysql2::Error => e
293
- raise "mysql_replicator_multi: #{e}"
305
+ if config['enable_retry'] == 1
306
+ raise e
307
+ else
308
+ raise "mysql_replicator_multi: #{e}"
309
+ end
294
310
  end
295
311
  end
296
312
  end
data/setup_mysql_replicator_multi.sql CHANGED
@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
29
29
  `enable_loose_insert` int(11) DEFAULT '0',
30
30
  -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
31
31
  `enable_loose_delete` int(11) DEFAULT '0',
32
+ -- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
33
+ `enable_retry` int(11) DEFAULT '1',
34
+ -- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
35
+ `retry_interval` int(11) NOT NULL DEFAULT '30',
32
36
  PRIMARY KEY (`id`),
33
37
  UNIQUE KEY `name` (`name`)
34
38
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql-replicator
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Yoshida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-10-27 00:00:00.000000000 Z
11
+ date: 2024-03-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake