fluent-plugin-mysql-replicator 0.4.3 → 0.5.0

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 
 rvm:
+  - 2.2
   - 2.1
   - 2.0.0
   - 1.9.3

data/example/mysql_multi_table_to_elasticsearch.md

@@ -28,6 +28,13 @@ It is a guide to replicate multiple mysql table to elasticsearch.
   host localhost
   port 9200
 
+  # You can configure to use SSL for connecting to Elasticsearch.
+  # ssl true
+
+  # Basic authentication credentials can be configured
+  # username basic_auth_username
+  # password basic_auth_password
+
   # Set Elasticsearch index, type, and unique id (primary_key) from tag.
   tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
 

data/example/mysql_single_table_to_elasticsearch.md

@@ -37,6 +37,13 @@ It is a guide to replicate single mysql table to elasticsearch.
   host localhost
   port 9200
 
+  # You can configure to use SSL for connecting to Elasticsearch.
+  # ssl true
+
+  # Basic authentication credentials can be configured
+  # username basic_auth_username
+  # password basic_auth_password
+
   # Set Elasticsearch index, type, and unique id (primary_key) from tag.
   tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
 

data/fluent-plugin-mysql-replicator.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-mysql-replicator"
-  s.version     = "0.4.3"
+  s.version     = "0.5.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
@@ -11,9 +11,11 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+  s.license = "Apache-2.0"
 
   s.add_development_dependency "rake"
   s.add_development_dependency "webmock"
+  s.add_development_dependency "test-unit", ">= 3.1.0"
 
   s.add_runtime_dependency "fluentd"
   s.add_runtime_dependency "mysql2"

data/lib/fluent/plugin/in_mysql_replicator.rb

@@ -2,6 +2,11 @@ module Fluent
   class MysqlReplicatorInput < Fluent::Input
     Plugin.register_input('mysql_replicator', self)
 
+    # Define `router` method to support v0.10.57 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Engine }
+    end
+
     def initialize
       require 'mysql2'
       require 'digest/sha1'
@@ -11,7 +16,7 @@ module Fluent
     config_param :host, :string, :default => 'localhost'
     config_param :port, :integer, :default => 3306
     config_param :username, :string, :default => 'root'
-    config_param :password, :string, :default => nil
+    config_param :password, :string, :default => nil, :secret => true
     config_param :database, :string, :default => nil
     config_param :encoding, :string, :default => 'utf8'
     config_param :query, :string
@@ -123,7 +128,7 @@ module Fluent
     end
 
     def emit_record(tag, record)
-      Engine.emit(tag, Engine.now, record)
+      router.emit(tag, Engine.now, record)
     end
 
     def query(query, con = nil)

data/lib/fluent/plugin/in_mysql_replicator_multi.rb

@@ -2,6 +2,11 @@ module Fluent
   class MysqlReplicatorMultiInput < Fluent::Input
     Plugin.register_input('mysql_replicator_multi', self)
 
+    # Define `router` method to support v0.10.57 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Engine }
+    end
+
     def initialize
       require 'mysql2'
       require 'digest/sha1'
@@ -11,7 +16,7 @@ module Fluent
     config_param :manager_host, :string, :default => 'localhost'
     config_param :manager_port, :integer, :default => 3306
     config_param :manager_username, :string, :default => nil
-    config_param :manager_password, :string, :default => ''
+    config_param :manager_password, :string, :default => '', :secret => true
     config_param :manager_database, :string, :default => 'replicator_manager'
     config_param :tag, :string, :default => nil
 
@@ -239,7 +244,7 @@ module Fluent
     end
 
     def emit_record(tag, record)
-      Engine.emit(tag, Engine.now, record)
+      router.emit(tag, Engine.now, record)
     end
 
     def get_manager_connection

data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb

@@ -7,6 +7,9 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 9200
   config_param :tag_format, :string, :default => nil
+  config_param :ssl, :bool, :default => false
+  config_param :username, :string, :default => nil
+  config_param :password, :string, :default => nil, :secret => true
 
   DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
 
@@ -60,7 +63,13 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
     bulk_message << ""
 
     http = Net::HTTP.new(@host, @port.to_i)
+    http.use_ssl = @ssl
+
     request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
+    if @username && @password
+      request.basic_auth(@username, @password)
+    end
+
     request.body = bulk_message.join("\n")
     http.request(request).value
   end

data/test/plugin/test_out_mysql_replicator_elasticsearch.rb

@@ -124,4 +124,35 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
       driver.run
     }
   end
-end
+
+  def test_writes_to_https_host
+    driver.configure("ssl true\n")
+    elastic_request = stub_elastic("https://localhost:9200/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_writes_to_http_basic_auth
+    driver.configure(%[
+      username foo\n
+      password bar\n
+    ])
+    elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_writes_to_http_basic_auth_failed
+    driver.configure(%[
+      username wront_user\n
+      password bar\n
+    ])
+    elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
+    driver.emit(sample_record)
+    assert_raise(WebMock::NetConnectNotAllowedError) {
+      driver.run
+    }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-replicator
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-18 00:00:00.000000000 Z
+date: 2015-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +138,8 @@ files:
 - test/plugin/test_out_mysql_replicator_elasticsearch.rb
 - test/plugin/test_out_mysql_replicator_solr.rb
 homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
-licenses: []
+licenses:
+- Apache-2.0
 post_install_message: 
 rdoc_options: []
 require_paths: