fluent-plugin-elasticsearch 1.10.0 → 1.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7f9583b7923380fe06d399cc24006cf75a17528
-  data.tar.gz: ffb1c9ed360ef6c48cfa003ee81022d179f0b9bf
+  metadata.gz: 6924c1c7c90f7eb629011bbae3db822799584200
+  data.tar.gz: 52ab17e9b443275e215de285327be011881681b9
 SHA512:
-  metadata.gz: 74eaadfe9d121623014befb052b8437cdb092a9857153c521039ee463ef51a20877f7f1e63dea4ad9c5f3a8f70ab33b0ea1c77cf0aa93fac0c33b0da63e15af6
-  data.tar.gz: 4a84606298bf3e2874485c6d5d10212f065d89f68ab7bc44e0731cfa3908a2cefce18aa9eef51f1b2fb09248e9a98d8610ebc5bd3496b9223cbb4245e00d6b04
+  metadata.gz: 85120102f59c4713273af0706b90e36a696e27329c63a3f748762c626633f02fe22ae60ebd6819c9f9819dcc4fa5d05d5df7ed58f2d4e43a3e2625be7599559d
+  data.tar.gz: 4f0a559e177c0fccaf0e1bb916decf0226508b49af463448556b76622c5c9ba064246052da1b92fb876d36ce77f0e3644f18144f3cd8b7e72fc7fae6ed70737d

data/History.md

@@ -4,6 +4,10 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.10.1
+- backport escaping basic authentication user information placeholders (#309)
+- backport handling dynamic config misconfiguration (#308)
+
 ### 1.10.0
 - backport adding `logstash_prefix_separator` parameter fix
 - backport making configuraable SSL/TLS version (#300)

data/README.md

@@ -94,6 +94,12 @@ You can specify multiple ElasticSearch hosts with separator ",".
 
 If you specify multiple hosts, this plugin will load balance updates to ElasticSearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
 
+And this plugin will escape required URL encoded characters within `%{}` placeholders.
+
+```
+hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+```
+
 ### user, password, path, scheme, ssl_verify
 
 If you specify this option, host and port options are ignored.
@@ -107,6 +113,13 @@ scheme https
 
 You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
 
+And this plugin will escape required URL encoded characters within `%{}` placeholders.
+
+```
+user %{demo+}
+password %{@secret}
+```
+
 Specify `ssl_verify false` to skip ssl verification (defaults to true)
 
 ### logstash_format

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.10.0'
+  s.version       = '1.10.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'webmock', '~> 1'
   s.add_development_dependency 'test-unit', '~> 3.1.0'
   s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'flexmock', '~> 2.3.5'
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -105,6 +105,13 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     rescue LoadError
       @dump_proc = Yajl.method(:dump)
     end
+
+    if @user && m = @user.match(/%{(?<user>.*)}/)
+      @user = URI.encode_www_form_component(m["user"])
+    end
+    if @password && m = @password.match(/%{(?<password>.*)}/)
+      @password = URI.encode_www_form_component(m["password"])
+    end
   end
 
   def create_meta_config_map
@@ -172,6 +179,18 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     end
   end
 
+  def get_escaped_userinfo(host_str)
+    if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
+      m["scheme"] +
+        URI.encode_www_form_component(m["user"]) +
+        ':' +
+        URI.encode_www_form_component(m["password"]) +
+        m["path"]
+    else
+      host_str
+    end
+  end
+
   def get_connection_options
     raise "`password` must be present if `user` is present" if @user && !@password
 
@@ -186,7 +205,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
           }
         else
           # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
-          uri = URI(host_str)
+          uri = URI(get_escaped_userinfo(host_str))
           %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
             hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
             hash

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -80,7 +80,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
           }
         else
           # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
-          uri = URI(host_str)
+          uri = URI(get_escaped_userinfo(host_str))
           %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
             hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
             hash
@@ -121,17 +121,23 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     chunk.msgpack_each do |time, record|
       next unless record.is_a? Hash
 
-      # evaluate all configurations here
-      DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
-        k = DYNAMIC_PARAM_NAMES[i]
-        v = self.instance_variable_get(var)
-        # check here to determine if we should evaluate
-        if dynamic_conf[k] != v
-          value = expand_param(v, tag, time, record)
-          dynamic_conf[k] = value
-        end
-      }
+      begin
+        # evaluate all configurations here
+        DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
+          k = DYNAMIC_PARAM_NAMES[i]
+          v = self.instance_variable_get(var)
+          # check here to determine if we should evaluate
+          if dynamic_conf[k] != v
+            value = expand_param(v, tag, time, record)
+            dynamic_conf[k] = value
+          end
+        }
       # end eval all configs
+      rescue => e
+        # handle dynamic parameters misconfigurations
+        router.emit_error_event(tag, time, record, e)
+        next
+      end
 
       if eval_or_val(dynamic_conf['logstash_format'])
         if record.has_key?("@timestamp")

data/test/plugin/test_out_elasticsearch.rb

@@ -311,6 +311,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_hosts_list_with_escape_placeholders
+    config = %{
+      hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'passw%40rd', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -330,6 +356,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal nil, host1[:path]
   end
 
+  def test_single_host_params_and_defaults_with_escape_placeholders
+    config = %{
+      host     logs.google.com
+      user     %{j+hn}
+      password %{d@e}
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 1, instance.get_connection_options[:hosts].length
+    host1 = instance.get_connection_options[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'd%40e', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -1,7 +1,10 @@
 require 'helper'
 require 'date'
+require 'flexmock/test_unit'
 
 class ElasticsearchOutputDynamic < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   attr_accessor :index_cmds, :index_command_counts
 
   def setup
@@ -139,6 +142,32 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_hosts_list_with_escape_placeholders
+    config = %{
+      hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 2, instance.get_connection_options(nil)[:hosts].length
+    host1, host2 = instance.get_connection_options(nil)[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'passw%40rd', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -158,6 +187,25 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal nil, host1[:path]
   end
 
+  def test_single_host_params_and_defaults_with_escape_placeholders
+    config = %{
+      host     logs.google.com
+      user     %{j+hn}
+      password %{d@e}
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 1, instance.get_connection_options(nil)[:hosts].length
+    host1 = instance.get_connection_options(nil)[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'd%40e', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
@@ -537,6 +585,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run
   end
 
+  def test_tag_parts_index_error_event
+    stub_elastic_ping
+    stub_elastic
+    driver.configure("logstash_prefix ${tag_parts[1]}\n")
+    flexmock(driver.instance.router).should_receive(:emit_error_event)
+      .with('test', Integer, Hash, TypeError).once
+    driver.emit(sample_record)
+    driver.run
+  end
+
   def test_connection_failed_retry
     connection_resets = 0
 
@@ -566,7 +624,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
-    
+
     driver.configure("reconnect_on_error true\n")
     driver.emit(sample_record)
 
@@ -590,7 +648,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
-    
+
     driver.configure("reconnect_on_error false\n")
     driver.emit(sample_record)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.10.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-25 00:00:00.000000000 Z
+date: 2017-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -109,6 +109,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.8'
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.5
 description: ElasticSearch output plugin for Fluent event collector
 email:
 - pitr.vern@gmail.com
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector