fluent-plugin-mysql-replicator 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c523ddf1d5052d11c6c7a0f7e9172d319c881257d791a0006cd1a6601f5a948
-  data.tar.gz: ca9f218dfa56543f141d8911578250194281bfbad16d8f5ed6b4980120d99c8d
+  metadata.gz: 76bd65971409a89e44f0cc67010a9365b1d78bd195d68bb1ddffd90294b6059c
+  data.tar.gz: 27f0bedbb9814a9e08d36bdc53bde79c142465cbb928629695464ebfe25150af
 SHA512:
-  metadata.gz: a5266cd4f824e781d53eaa613c37f260c7814badcee87fb6c49a1321e9c9ac8bc8d9505b2ea800f3a8ccde237b2b0eeb808e187463996c2b1aecc93065128869
-  data.tar.gz: 70834ef8629d34bbd53aa1ff9d66ba706d6df5b48cd310d8a554c7d25628682633860123f63855e851482abda385e6288c4fcd01da848cbffc56bc0525553968
+  metadata.gz: b6eb13f3bb9da56d5de5b474388c40582de0cb06412eb1d83c8bd635f979dc88c4464ff8e570998f16d458db9746d1e0eb1a312e1648215474b2991a622e7684
+  data.tar.gz: a1bd184c16ca5d2a730884a52db60977510302f40d11b7f41c52e2e6cbaa5a93e8e208d0c1371276bd5b7cee839561e8676b4cb63f7657133695c2d667374897

data/README.md

@@ -31,13 +31,13 @@ install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator -v 1.0.2
+$ gem install fluent-plugin-mysql-replicator -v 1.0.3
 
 # for td-agent2
 $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
 
 # for td-agent3
-$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.2
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
 `````
 
 ## Included plugins

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.0.2"
+  s.version     = "1.0.3"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -85,37 +85,49 @@ module Fluent::Plugin
         while @running
           rows_count = 0
           start_time = Time.now
-          unless config['prepared_query'].nil?
-            nest_db = get_origin_connection(config)
-            config['prepared_query'].strip.split(/;/).each do |query|
-              nest_db.query(query)
+          db = nil
+          nest_db = nil
+          begin
+            unless config['prepared_query'].nil?
+              nest_db = get_origin_connection(config)
+              config['prepared_query'].strip.split(/;/).each do |query|
+                nest_db.query(query)
+              end
             end
-          end
-          db = get_origin_connection(config)
-          db.query(config['query']).each do |row|
-            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-            row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
-              row[k] = [] unless row[k].is_a?(Array)
-              nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
-                nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-                row[k] << nest_row
+            db = get_origin_connection(config)
+            db.query(config['query']).each do |row|
+              row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
+                row[k] = [] unless row[k].is_a?(Array)
+                nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
+                  nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+                  row[k] << nest_row
+                end
               end
+              current_id = row[primary_key]
+              @mutex.synchronize {
+                if row[primary_key].nil?
+                  log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
+                  break
+                end
+                detect_insert_update(config, row)
+                detect_delete(config, current_id, previous_id)
+              }
+              previous_id = current_id
+              rows_count += 1
             end
-            current_id = row[primary_key]
+          rescue Mysql2::Error => e
+            raise e unless config['enable_retry'] == 1
+
             @mutex.synchronize {
-              if row[primary_key].nil?
-                log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
-                break
-              end
-              detect_insert_update(config, row)
-              detect_delete(config, current_id, previous_id)
+              log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
+              log.error "error: #{e.message}"
+              log.error e.backtrace.join("\n")
             }
-            previous_id = current_id
-            rows_count += 1
-          end
-          db.close
-          unless config['prepared_query'].nil?
-            nest_db.close
+            sleep config['retry_interval']
+          ensure
+            db.close if db
+            nest_db.close if nest_db && !config['prepared_query'].nil?
           end
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
@@ -290,7 +302,11 @@ module Fluent::Plugin
           :cache_rows => false
         )
       rescue Mysql2::Error => e
-        raise "mysql_replicator_multi: #{e}"
+        if config['enable_retry'] == 1
+          raise e
+        else
+          raise "mysql_replicator_multi: #{e}"
+        end
       end
     end
   end

data/setup_mysql_replicator_multi.sql

@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `enable_loose_insert` int(11) DEFAULT '0',
   -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
   `enable_loose_delete` int(11) DEFAULT '0',
+  -- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
+  `enable_retry` int(11) DEFAULT '1',
+  -- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
+  `retry_interval` int(11) NOT NULL DEFAULT '30',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2024-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake