fluent-plugin-mysql-replicator 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f457b520cc136f8ec0b3c78ee690fa20e4ce65a9
-  data.tar.gz: 2df7a1c2c1bbf24741b6fbf9ed35b819d499eb80
+SHA256:
+  metadata.gz: 76bd65971409a89e44f0cc67010a9365b1d78bd195d68bb1ddffd90294b6059c
+  data.tar.gz: 27f0bedbb9814a9e08d36bdc53bde79c142465cbb928629695464ebfe25150af
 SHA512:
-  metadata.gz: b96f9134fa44ca80468a77abbbc10f30f28f8a1c12cd60a91e56f0c1a4a9e8e4768d41faa67646f57ea3b045815eb753bf6e1a89bb92c5385faf2afdc6786f11
-  data.tar.gz: '088c4f52b9565348fde62f86232b23f315b268db9e5d52bdcda7055eca2167b837336e0b18ede444657c0c0acf1c2083de245dd15c15f18e9caa529f44ebcf70'
+  metadata.gz: b6eb13f3bb9da56d5de5b474388c40582de0cb06412eb1d83c8bd635f979dc88c4464ff8e570998f16d458db9746d1e0eb1a312e1648215474b2991a622e7684
+  data.tar.gz: a1bd184c16ca5d2a730884a52db60977510302f40d11b7f41c52e2e6cbaa5a93e8e208d0c1371276bd5b7cee839561e8676b4cb63f7657133695c2d667374897

data/README.md

@@ -31,13 +31,13 @@ install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator -v 1.0.1
+$ gem install fluent-plugin-mysql-replicator -v 1.0.3
 
 # for td-agent2
 $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
 
 # for td-agent3
-$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.1
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.3
 `````
 
 ## Included plugins

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "1.0.1"
+  s.version     = "1.0.3"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -1,9 +1,11 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 
 module Fluent::Plugin
-  class MysqlReplicatorMultiInput < Fluent::Input
+  class MysqlReplicatorMultiInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('mysql_replicator_multi', self)
 
+    helpers :thread
+
     def initialize
       require 'mysql2'
       require 'digest/sha1'
@@ -34,6 +36,7 @@ module Fluent::Plugin
         @mutex = Mutex.new
         @manager_db = get_manager_connection
         @manager_db.query("SET SESSION wait_timeout=1800;")
+        @running = true
         @threads << thread_create(:in_mysql_replicator_flusher) {
           @hash_table_bulk_insert = []
           @hash_table_bulk_insert_last_time = Time.now
@@ -51,7 +54,13 @@ module Fluent::Plugin
       end
     end
 
+    def stop
+      @running = false
+      super
+    end
+
     def shutdown
+      @threads.each(&:join)
       super
     end
 
@@ -73,40 +82,52 @@ module Fluent::Plugin
         }
         primary_key = config['primary_key']
         previous_id = current_id = nil
-        loop do
+        while @running
           rows_count = 0
           start_time = Time.now
-          unless config['prepared_query'].nil?
-            nest_db = get_origin_connection(config)
-            config['prepared_query'].strip.split(/;/).each do |query|
-              nest_db.query(query)
+          db = nil
+          nest_db = nil
+          begin
+            unless config['prepared_query'].nil?
+              nest_db = get_origin_connection(config)
+              config['prepared_query'].strip.split(/;/).each do |query|
+                nest_db.query(query)
+              end
             end
-          end
-          db = get_origin_connection(config)
-          db.query(config['query']).each do |row|
-            row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-            row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
-              row[k] = [] unless row[k].is_a?(Array)
-              nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
-                nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-                row[k] << nest_row
+            db = get_origin_connection(config)
+            db.query(config['query']).each do |row|
+              row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+              row.select {|k, v| v.to_s.strip.match(/^SELECT[^\$]+\$\{[^\}]+\}/i) }.each do |k, v|
+                row[k] = [] unless row[k].is_a?(Array)
+                nest_db.query(v.gsub(/\$\{([^\}]+)\}/) {|matched| row[$1].to_s}).each do |nest_row|
+                  nest_row.each {|k, v| nest_row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
+                  row[k] << nest_row
+                end
               end
+              current_id = row[primary_key]
+              @mutex.synchronize {
+                if row[primary_key].nil?
+                  log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
+                  break
+                end
+                detect_insert_update(config, row)
+                detect_delete(config, current_id, previous_id)
+              }
+              previous_id = current_id
+              rows_count += 1
             end
-            current_id = row[primary_key]
+          rescue Mysql2::Error => e
+            raise e unless config['enable_retry'] == 1
+
             @mutex.synchronize {
-              if row[primary_key].nil?
-                log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
-                break
-              end
-              detect_insert_update(config, row)
-              detect_delete(config, current_id, previous_id)
+              log.error "mysql_replicator_multi: failed due to an error caused by the database. :setting_name=>#{config['name']}"
+              log.error "error: #{e.message}"
+              log.error e.backtrace.join("\n")
             }
-            previous_id = current_id
-            rows_count += 1
-          end
-          db.close
-          unless config['prepared_query'].nil?
-            nest_db.close
+            sleep config['retry_interval']
+          ensure
+            db.close if db
+            nest_db.close if nest_db && !config['prepared_query'].nil?
           end
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
@@ -214,7 +235,7 @@ module Fluent::Plugin
 
     def hash_table_flusher
       begin
-        loop do
+        while @running
           if @hash_table_bulk_insert.empty? || @bulk_insert_timeout > (Time.now - @hash_table_bulk_insert_last_time)
             sleep @bulk_insert_timeout
             next
@@ -223,6 +244,9 @@ module Fluent::Plugin
             flush_hash_table
           }
         end
+        @mutex.synchronize {
+          flush_hash_table
+        }
       rescue StandardError => e
         @mutex.synchronize {
           log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
@@ -278,7 +302,11 @@ module Fluent::Plugin
           :cache_rows => false
         )
       rescue Mysql2::Error => e
-        raise "mysql_replicator_multi: #{e}"
+        if config['enable_retry'] == 1
+          raise e
+        else
+          raise "mysql_replicator_multi: #{e}"
+        end
       end
     end
   end

data/setup_mysql_replicator_multi.sql

@@ -29,6 +29,10 @@ CREATE TABLE IF NOT EXISTS `settings` (
   `enable_loose_insert` int(11) DEFAULT '0',
   -- On enabling 'enable_loose_delete: 1', turn on speculative delete but performance penalty on non-contiguous primary key.
   `enable_loose_delete` int(11) DEFAULT '0',
+  -- On enabling 'enable_retry: 1', automatically retries when an error occurs due to MySQL.
+  `enable_retry` int(11) DEFAULT '1',
+  -- Additional interval when retrying. If not set, waits for the time set in the regular interval column.
+  `retry_interval` int(11) NOT NULL DEFAULT '30',
   PRIMARY KEY (`id`),
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.3
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2024-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -149,11 +149,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database
   server. Not only that, it could multiple table replication and generate nested document
   for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_in_mysql_replicator.rb
+- test/plugin/test_in_mysql_replicator_multi.rb
+- test/plugin/test_out_mysql_replicator_elasticsearch.rb
+- test/plugin/test_out_mysql_replicator_solr.rb