fluent-plugin-mysql-replicator 0.6.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e0d8447f15a51520238bda787d9f00e1ffc2b17
-  data.tar.gz: acd39ee14e5eca137189069f652b91baa23847e4
+  metadata.gz: e43736658fa9d53723c3f1a2089c46108016bc68
+  data.tar.gz: 187e611b52bfb740337041716f68cddfa879b433
 SHA512:
-  metadata.gz: 2c8d1b8c80419b6b7b88540181b7a5d2c8b4cb8b5d7dca7489cfe9e163fdfa225cbc27ace78c31ffa7b198a54447ca1802a68c907d1a201ffd2e94b9c47123e1
-  data.tar.gz: 1465dbf6119e9a07ca3e177d2825868fa34fe0f4382c1c676f004a38493dcb1fba4c97e52f4b6991273b49dda9f2e8210cd7ebc2ff84138c02d44c363b4ee10e
+  metadata.gz: c89083833cf2cf8e2ae7dadd472227744806700ba59803474b2f3d7a0451a3d1bb996014039a5ef75f2e4d3817403fd6fc13efac45ba03b0597f30b25bb75a8b
+  data.tar.gz: f127d31178d16bba47b133a537fb1f8f1c2df1813444b05898f712d97914bd8861f5381fa4a1c4ef6a2f7392980c908364ae6a6a7673aad5a825074a1addda61

data/README.md

@@ -10,8 +10,8 @@ It's comming support replicate to another RDB/noSQL.
 
 | fluent-plugin-mysql-replicator | fluentd    | ruby   |
 |--------------------|------------|--------|
-|  0.6.2            | v0.14.x | >= 2.1 |
-|  0.6.2            | v0.12.x | >= 1.9 |
+|  >= 0.6.1          | >= v0.14.x | >= 2.1 |
+|  <= 0.6.1          | >= v0.12.x | >= 1.9 |
 
 ## Dependency
 
@@ -31,10 +31,10 @@ install with gem or fluent-gem command as:
 
 `````
 # for system installed fluentd
-$ gem install fluent-plugin-mysql-replicator -v 0.6.2
+$ gem install fluent-plugin-mysql-replicator -v 0.6.1
 
 # for td-agent2
-$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.2
+$ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
 `````
 
 ## Included plugins

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.6.2"
+  s.version     = "1.0.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "webmock", "~> 1.24.0"
   s.add_development_dependency "test-unit", ">= 3.1.0"
 
-  s.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
+  s.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
   s.add_runtime_dependency "mysql2"
   s.add_runtime_dependency "rsolr"
 end

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -1,14 +1,12 @@
-require 'fluent/input'
+require 'mysql2'
+require 'digest/sha1'
+require 'fluent/plugin/input'
 
-module Fluent
-  class MysqlReplicatorInput < Fluent::Input
-    Plugin.register_input('mysql_replicator', self)
+module Fluent::Plugin
+  class MysqlReplicatorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('mysql_replicator', self)
 
-    def initialize
-      require 'mysql2'
-      require 'digest/sha1'
-      super
-    end
+    helpers :thread
 
     config_param :host, :string, :default => 'localhost'
     config_param :port, :integer, :default => 3306
@@ -25,7 +23,7 @@ module Fluent
 
     def configure(conf)
       super
-      @interval = Config.time_value(@interval)
+      @interval = Fluent::Config.time_value(@interval)
 
       if @tag.nil?
         raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
@@ -35,11 +33,12 @@ module Fluent
     end
 
     def start
-      @thread = Thread.new(&method(:run))
+      super
+      thread_create(:in_mysql_replicator_runner, &method(:run))
     end
 
     def shutdown
-      Thread.kill(@thread)
+     super
     end
 
     def run
@@ -107,7 +106,7 @@ module Fluent
           end
           if deleted_ids.count > 0
             hash_delete_by_list(table_hash, deleted_ids)
-            deleted_ids.each do |id| 
+            deleted_ids.each do |id|
               tag = format_tag(@tag, {:event => :delete})
               emit_record(tag, {@primary_key => id})
             end

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -1,8 +1,8 @@
 require 'fluent/input'
 
-module Fluent
+module Fluent::Plugin
   class MysqlReplicatorMultiInput < Fluent::Input
-    Plugin.register_input('mysql_replicator_multi', self)
+    Fluent::Plugin.register_input('mysql_replicator_multi', self)
 
     def initialize
       require 'mysql2'
@@ -28,18 +28,19 @@ module Fluent
     end
 
     def start
+      super
       begin
         @threads = []
         @mutex = Mutex.new
         @manager_db = get_manager_connection
         @manager_db.query("SET SESSION wait_timeout=1800;")
-        @threads << Thread.new {
+        @threads << thread_create(:in_mysql_replicator_flusher) {
           @hash_table_bulk_insert = []
           @hash_table_bulk_insert_last_time = Time.now
           hash_table_flusher
         }
-        get_settings.each do |config|
-          @threads << Thread.new {
+        get_settings.each_with_index do |config, idx|
+          @threads << thread_create(:"in_mysql_replicator_pollers_#{idx}") {
             poll(config)
           }
         end
@@ -51,9 +52,7 @@ module Fluent
     end
 
     def shutdown
-      @threads.each do |thread|
-        Thread.kill(thread)
-      end
+      super
     end
 
     def get_settings
@@ -106,9 +105,6 @@ module Fluent
             rows_count += 1
           end
           db.close
-          unless config['prepared_query'].nil?
-            nest_db.close
-          end
           elapsed_time = sprintf("%0.02f", Time.now - start_time)
           @mutex.synchronize {
             log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -1,9 +1,14 @@
 require 'net/http'
 require 'date'
+require 'fluent/plugin/output'
 
-class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
+class Fluent::Plugin::MysqlReplicatorElasticsearchOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('mysql_replicator_elasticsearch', self)
 
+  DEFAULT_BUFFER_TYPE = "memory"
+
+  helpers :compat_parameters
+
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 9200
   config_param :tag_format, :string, :default => nil
@@ -11,6 +16,10 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
   config_param :username, :string, :default => nil
   config_param :password, :string, :default => nil, :secret => true
 
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+  end
+
   DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
   def initialize
@@ -39,6 +48,14 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
     super
   end
 
+  def multi_workers_ready?
+    true
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
   def write(chunk)
     bulk_message = []
 

data/lib/fluent/plugin/out_mysql_replicator_solr.rb

@@ -1,13 +1,22 @@
 require 'rsolr'
 require 'uri'
+require 'fluent/plugin/output'
 
-class Fluent::MysqlReplicatorSolrOutput < Fluent::BufferedOutput
+class Fluent::Plugin::MysqlReplicatorSolrOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('mysql_replicator_solr', self)
 
+  DEFAULT_BUFFER_TYPE = "memory"
+
+  helpers :compat_parameters
+
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 8983
   config_param :tag_format, :string, :default => nil
 
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+  end
+
   DEFAULT_TAG_FORMAT = /(?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
   def initialize
@@ -36,6 +45,14 @@ class Fluent::MysqlReplicatorSolrOutput < Fluent::BufferedOutput
     super
   end
 
+  def multi_workers_ready?
+    true
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
   def write(chunk)
     solr_connection = {}
 

data/test/plugin/test_in_mysql_replicator.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
 class MysqlReplicatorInputTest < Test::Unit::TestCase
   def setup
@@ -14,8 +15,8 @@ class MysqlReplicatorInputTest < Test::Unit::TestCase
     record_hostname yes
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorInput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MysqlReplicatorInput).configure(conf)
   end
 
   def test_configure

data/test/plugin/test_in_mysql_replicator_multi.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
 class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
   def setup
@@ -13,8 +14,8 @@ class MysqlReplicatorMultiInputTest < Test::Unit::TestCase
     tag               replicator.${name}.${event}.${primary_key}
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorMultiInput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MysqlReplicatorMultiInput).configure(conf)
   end
 
   def test_configure

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/output'
 require 'webmock/test_unit'
 
 WebMock.disable_net_connect!
@@ -9,10 +10,11 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
     @driver = nil
+    @tag = 'myindex.mytype.insert.id'
   end
 
-  def driver(tag='myindex.mytype.insert.id', conf='')
-    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::MysqlReplicatorElasticsearchOutput, tag).configure(conf)
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlReplicatorElasticsearchOutput).configure(conf)
   end
 
   def sample_record
@@ -32,56 +34,63 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
 
   def test_wrties_with_proper_content_type
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal("application/json; charset=utf-8", @content_type)
   end
 
   def test_writes_to_speficied_index
     driver.configure("index_name myindex\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_speficied_type
     driver.configure("type_name mytype\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
   def test_writes_to_speficied_port
     driver.configure("port 9201\n")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
   def test_makes_bulk_request
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(4, index_cmds.count)
   end
 
   def test_all_records_are_preserved_in_bulk
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(26, index_cmds[1]['age'])
     assert_equal(27, index_cmds[3]['age'])
   end
@@ -89,47 +98,53 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['@timestamp'])
   end
 
 
   def test_doesnt_add_tag_key_by_default
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['tag'])
   end
 
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
   def test_adds_id_key_when_not_configured
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
   def test_request_error
     stub_elastic_unavailable
-    driver.emit(sample_record)
     assert_raise(Net::HTTPFatalError) {
-      driver.run
+      driver.run(default_tag: @tag) do
+        driver.feed(sample_record)
+      end
     }
   end
 
   def test_writes_to_https_host
     driver.configure("ssl true\n")
     elastic_request = stub_elastic("https://localhost:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -139,8 +154,9 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
       password bar\n
     ])
     elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: @tag) do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -150,9 +166,10 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
       password bar\n
     ])
     elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
-    driver.emit(sample_record)
     assert_raise(WebMock::NetConnectNotAllowedError) {
-      driver.run
+      driver.run(default_tag: @tag) do
+        driver.feed(sample_record)
+      end
     }
   end
 end

data/test/plugin/test_out_mysql_replicator_solr.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/output'
 
 class MysqlReplicatorSolrOutput < Test::Unit::TestCase
 
@@ -12,8 +13,8 @@ class MysqlReplicatorSolrOutput < Test::Unit::TestCase
     tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::MysqlReplicatorSolrOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MysqlReplicatorSolrOutput).configure(conf)
   end
 
   def test_configure

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2019-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'