fluent-plugin-mysql-replicator 0.6.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ffa1646346a6d10f15cc8837a0551ecdcddfd0c0
-  data.tar.gz: 5a3f350ffdeda01067e3b7ab90b35078333f9ee6
+SHA256:
+  metadata.gz: 5c523ddf1d5052d11c6c7a0f7e9172d319c881257d791a0006cd1a6601f5a948
+  data.tar.gz: ca9f218dfa56543f141d8911578250194281bfbad16d8f5ed6b4980120d99c8d
 SHA512:
-  metadata.gz: d96f0e8226cb824bf44a14d961528cef03a3831df71a0a5a6810d68935733fbe1f940d44875ef7871f2b757845174e168bd69fd77239fad96f773ef967b99af2
-  data.tar.gz: 2c5886bdaa8050b70024cfeab747bdadaa6c0eb22bbb4b431aad818502969f58a3fbd23315565d41d74c4b9ee7cd9c9fca0f22681aa93c30dececa78647d5a22
+  metadata.gz: a5266cd4f824e781d53eaa613c37f260c7814badcee87fb6c49a1321e9c9ac8bc8d9505b2ea800f3a8ccde237b2b0eeb808e187463996c2b1aecc93065128869
+  data.tar.gz: 70834ef8629d34bbd53aa1ff9d66ba706d6df5b48cd310d8a554c7d25628682633860123f63855e851482abda385e6288c4fcd01da848cbffc56bc0525553968

data/.travis.yml

@@ -7,6 +7,7 @@ rvm:
   - 2.1
 
 services:
+  - mysql
   - elasticsearch
 
 before_install:

data/README.md

@@ -10,8 +10,20 @@ It's comming support replicate to another RDB/noSQL.
 
 | fluent-plugin-mysql-replicator | fluentd    | ruby   |
 |--------------------|------------|--------|
-|  0.6.1            | v0.14.x | >= 2.1 |
-|  0.6.1            | v0.12.x | >= 1.9 |
+|  >= 0.6.1          | >= v0.14.x | >= 2.1 |
+|  <= 0.6.1          | >= v0.12.x | >= 1.9 |
+
+## Dependency
+
+before use, install dependent library as:
+
+```bash
+# for RHEL/CentOS
+$ sudo yum group install "Development Tools"
+
+# for Ubuntu/Debian
+$ sudo apt-get install build-essential
+```
 
 ## Installation
 
@@ -19,10 +31,13 @@ install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator -v 0.6.1
+$ gem install fluent-plugin-mysql-replicator -v 1.0.2
 
 # for td-agent2
 $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
+
+# for td-agent3
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 1.0.2
 `````
 
 ## Included plugins

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.6.1"
+  s.version     = "1.0.2"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "webmock", "~> 1.24.0"
   s.add_development_dependency "test-unit", ">= 3.1.0"
 
-  s.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
+  s.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
   s.add_runtime_dependency "mysql2"
   s.add_runtime_dependency "rsolr"
 end

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -1,14 +1,12 @@
-require 'fluent/input'
+require 'mysql2'
+require 'digest/sha1'
+require 'fluent/plugin/input'
 
-module Fluent
-  class MysqlReplicatorInput < Fluent::Input
-    Plugin.register_input('mysql_replicator', self)
+module Fluent::Plugin
+  class MysqlReplicatorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('mysql_replicator', self)
 
-    def initialize
-      require 'mysql2'
-      require 'digest/sha1'
-      super
-    end
+    helpers :thread
 
     config_param :host, :string, :default => 'localhost'
     config_param :port, :integer, :default => 3306
@@ -25,7 +23,7 @@ module Fluent
 
     def configure(conf)
       super
-      @interval = Config.time_value(@interval)
+      @interval = Fluent::Config.time_value(@interval)
 
       if @tag.nil?
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
@@ -35,11 +33,12 @@ module Fluent
     end
 
     def start
-      @thread = Thread.new(&method(:run))
+      super
+      thread_create(:in_mysql_replicator_runner, &method(:run))
     end
 
     def shutdown
-      Thread.kill(@thread)
+     super
     end
 
     def run
@@ -72,7 +71,7 @@ module Fluent
           current_ids << row[@primary_key]
           current_hash = Digest::SHA1.hexdigest(row.flatten.join)
           row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
-          row.select {|k, v| v.to_s.strip.match(/^SELECT/i) }.each do |k, v|
+          row.select {|k, v| v.to_s.strip.match(/^SELECT(\s+)/i) }.each do |k, v|
             row[k] = [] unless row[k].is_a?(Array)
             nest_rows, prepared_con = query(v.gsub(/\$\{([^\}]+)\}/, row[$1].to_s), prepared_con)
             nest_rows.each do |nest_row|
@@ -107,7 +106,7 @@ module Fluent
           end
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
-            deleted_ids.each do |id| 
+            deleted_ids.each do |id|
               tag = format_tag(@tag, {:event => :delete})
               emit_record(tag, {@primary_key => id})
             end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -1,8 +1,10 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
-  class MysqlReplicatorMultiInput < Fluent::Input
-    Plugin.register_input('mysql_replicator_multi', self)
+module Fluent::Plugin
+  class MysqlReplicatorMultiInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('mysql_replicator_multi', self)
+
+    helpers :thread
 
     def initialize
       require 'mysql2'
@@ -28,18 +30,20 @@ module Fluent
     end
 
     def start
+      super
       begin
         @threads = []
         @mutex = Mutex.new
         @manager_db = get_manager_connection
         @manager_db.query("SET SESSION wait_timeout=1800;")
-        @threads << Thread.new {
+        @running = true
+        @threads << thread_create(:in_mysql_replicator_flusher) {
           @hash_table_bulk_insert = []
           @hash_table_bulk_insert_last_time = Time.now
           hash_table_flusher
         }
-        get_settings.each do |config|
-          @threads << Thread.new {
+        get_settings.each_with_index do |config, idx|
+          @threads << thread_create(:"in_mysql_replicator_pollers_#{idx}") {
             poll(config)
           }
         end
@@ -50,10 +54,14 @@ module Fluent
       end
     end
 
+    def stop
+      @running = false
+      super
+    end
+
     def shutdown
-      @threads.each do |thread|
-        Thread.kill(thread)
-      end
+      @threads.each(&:join)
+      super
     end
 
     def get_settings
@@ -74,7 +82,7 @@ module Fluent
         }
         primary_key = config['primary_key']
         previous_id = current_id = nil
-        loop do
+        while @running
           rows_count = 0
           start_time = Time.now
           unless config['prepared_query'].nil?
@@ -106,6 +114,9 @@ module Fluent
             rows_count += 1
           end
           db.close
+          unless config['prepared_query'].nil?
+            nest_db.close
+          end
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
             log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
@@ -212,7 +223,7 @@ module Fluent
 
     def hash_table_flusher
       begin
-        loop do
+        while @running
           if @hash_table_bulk_insert.empty? || @bulk_insert_timeout > (Time.now - @hash_table_bulk_insert_last_time)
             sleep @bulk_insert_timeout
             next
@@ -221,6 +232,9 @@ module Fluent
             flush_hash_table
           }
         end
+        @mutex.synchronize {
+          flush_hash_table
+        }
       rescue StandardError => e
         @mutex.synchronize {
           log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -1,9 +1,14 @@
 require 'net/http'
 require 'date'
+require 'fluent/plugin/output'
 
-class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
+class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('mysql_replicator_elasticsearch', self)
 
+  DEFAULT_BUFFER_TYPE = "memory"
+
+  helpers :compat_parameters
+
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 9200
   config_param :tag_format, :string, :default => nil
@@ -11,6 +16,10 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
   config_param :username, :string, :default => nil
   config_param :password, :string, :default => nil, :secret => true
 
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+  end
+
   DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
   def initialize
@@ -39,6 +48,14 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
     super
   end
 
+  def multi_workers_ready?
+    true
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
   def write(chunk)
     bulk_message = []
 

data/lib/fluent/plugin/out_mysql_replicator_solr.rb

@@ -1,13 +1,22 @@
 require 'rsolr'
 require 'uri'
+require 'fluent/plugin/output'
 
-class Fluent::MysqlReplicatorSolrOutput < Fluent::BufferedOutput
+class Fluent::Plugin::MysqlReplicatorSolrOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('mysql_replicator_solr', self)
 
+  DEFAULT_BUFFER_TYPE = "memory"
+
+  helpers :compat_parameters
+
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 8983
   config_param :tag_format, :string, :default => nil
 
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+  end
+
   DEFAULT_TAG_FORMAT = /(?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
   def initialize
@@ -36,6 +45,14 @@ class Fluent::MysqlReplicatorSolrOutput < Fluent::BufferedOutput
     super
   end
 
+  def multi_workers_ready?
+    true
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
   def write(chunk)
     solr_connection = {}
 

data/test/plugin/test_in_mysql_replicator.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
 class MysqlReplicatorInputTest < Test::Unit::TestCase
   def setup
@@ -14,8 +15,8 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
     record_hostname yes
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorInput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MysqlReplicatorInput).configure(conf)
   end
 
   def test_configure

data/test/plugin/test_in_mysql_replicator_multi.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
 class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
   def setup
@@ -13,8 +14,8 @@ class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
     tag               replicator.${name}.${event}.${primary_key}
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorMultiInput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MysqlReplicatorMultiInput).configure(conf)
   end
 
   def test_configure

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/output'
 require 'webmock/test_unit'
 
 WebMock.disable_net_connect!
@@ -9,10 +10,11 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
     @driver = nil
+    @tag = 'myindex.mytype.insert.id'
   end
 
-  def driver(tag='myindex.mytype.insert.id', conf='')
-    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlReplicatorElasticsearchOutput, tag).configure(conf)
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlReplicatorElasticsearchOutput).configure(conf)
   end
 
   def sample_record
@@ -32,56 +34,63 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
 
   def test_wrties_with_proper_content_type
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal("application/json; charset=utf-8", @content_type)
   end
 
   def test_writes_to_speficied_index
     driver.configure("index_name myindex\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
   def test_writes_to_speficied_port
     driver.configure("port 9201\n")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
   def test_makes_bulk_request
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(4, index_cmds.count)
   end
 
   def test_all_records_are_preserved_in_bulk
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(26, index_cmds[1]['age'])
     assert_equal(27, index_cmds[3]['age'])
   end
@@ -89,47 +98,53 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['@timestamp'])
   end
 
 
   def test_doesnt_add_tag_key_by_default
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['tag'])
   end
 
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
   def test_adds_id_key_when_not_configured
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
   def test_request_error
     stub_elastic_unavailable
-    driver.emit(sample_record)
     assert_raise(Net::HTTPFatalError) {
-      driver.run
+      driver.run(default_tag: @tag) do
+        driver.feed(sample_record)
+      end
     }
   end
 
   def test_writes_to_https_host
     driver.configure("ssl true\n")
     elastic_request = stub_elastic("https://localhost:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -139,8 +154,9 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
       password bar\n
     ])
     elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -150,9 +166,10 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
       password bar\n
     ])
     elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
-    driver.emit(sample_record)
     assert_raise(WebMock::NetConnectNotAllowedError) {
-      driver.run
+      driver.run(default_tag: @tag) do
+        driver.feed(sample_record)
+      end
     }
   end
 end

data/test/plugin/test_out_mysql_replicator_solr.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/output'
 
 class MysqlReplicatorSolrOutput < Test::Unit::TestCase
 
@@ -12,8 +13,8 @@ class MysqlReplicatorSolrOutput < Test::Unit::TestCase
     tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorSolrOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlReplicatorSolrOutput).configure(conf)
   end
 
   def test_configure

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-15 00:00:00.000000000 Z
+date: 2021-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -149,8 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Fluentd input plugin to track insert/update/delete event from MySQL database