fluent-plugin-elasticsearch 2.0.0.rc.5 → 2.0.0.rc.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cc6bc3f6b89a0735bf96c5c0575d39e58458746
-  data.tar.gz: ee98c63584d8249795b5e4c1e1a75fcc2059b2ab
+  metadata.gz: ac60b728ec23b3ba9a24a87f1b58fcf3dbf2376f
+  data.tar.gz: fe9d1c7d1f5c07d84a69e27bc534c6c6db5512b6
 SHA512:
-  metadata.gz: eb6c6cb81709dc0b7aa2845e64346621db2cfed45ff16d75cb2384e669cfe8e0497efb8b1d36ea94dbc82da40397347662c118afeaef91347f30b73d93ec940f
-  data.tar.gz: e7eef17ef1e8cab63f79baebba312410455c997c1a5a2128416b48b5bf26c47320c93835e70167b8e218dc022b9171b9d7e802653e205c76dfc81018cfbf3689
+  metadata.gz: 51b2bf67047821e1d028acdc507d9f1f235444cd6ffa2d0f6dded6284231ef86473aed7e2e448daa89af31ff3a6cbb2a05aa17a36be5a8717a9a1473ad05941d
+  data.tar.gz: b5a8317b90ddd6b52a8c6cdd2fddf7f609682422bcd24308332d1a760acf990851fb3bfde16d7c6ccd7c430aeb0f1b03186e163c21b35d4a00f4b1b87242ad57

data/History.md

@@ -4,6 +4,11 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.0.0.rc.6
+- Improve documentation (#304)
+- Handle dynamic_config misconfigurations (#305)
+- Escape basic authentication user information placeholders (#306)
+
 ### 2.0.0.rc.5
 - make configurable with `ssl_version` parameter (#299)
 - add `logstash_prefix_separator` config parameter (#297)

data/README.md

@@ -109,6 +109,12 @@ You can specify multiple ElasticSearch hosts with separator ",".
 
 If you specify multiple hosts, this plugin will load balance updates to ElasticSearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
 
+And this plugin will escape required URL encoded characters within `%{}` placeholders.
+
+```
+hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+```
+
 ### user, password, path, scheme, ssl_verify
 
 If you specify this option, host and port options are ignored.
@@ -122,6 +128,13 @@ scheme https
 
 You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
 
+And this plugin will escape required URL encoded characters within `%{}` placeholders.
+
+```
+user %{demo+}
+password %{@secret}
+```
+
 Specify `ssl_verify false` to skip ssl verification (defaults to true)
 
 ### logstash_format
@@ -132,6 +145,8 @@ logstash_format true # defaults to false
 
 This is meant to make writing data into ElasticSearch indices compatible to what [Logstash](https://www.elastic.co/products/logstash) calls them. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana). See logstash_prefix and logstash_dateformat to customize this index name pattern. The index name will be `#{logstash_prefix}-#{formated_date}`
 
+:warning: Setting this option to `true` will ignore the `index_name` setting. The default index name prefix is `logstash-`.
+
 ### logstash_prefix
 
 ```
@@ -185,7 +200,7 @@ For example, should your input plugin not include a `time_key` in the record but
 By default, when inserting records in [Logstash](https://www.elastic.co/products/logstash) format, `@timestamp` is dynamically created with the time at log ingestion. If you'd like to use a custom time, include an `@timestamp` with your record.
 
 ```
-{"@timestamp":"2014-04-07T000:00:00-00:00"}
+{"@timestamp": "2014-04-07T000:00:00-00:00"}
 ```
 
 You can specify an option `time_key` (like the option described in [tail Input Plugin](http://docs.fluentd.org/articles/in_tail)) to replace `@timestamp` key.
@@ -348,7 +363,7 @@ This will add the Fluentd tag in the JSON record. For instance, if you have a co
 The record inserted into ElasticSearch would be
 
 ```
-{"_key":"my.logs", "name":"Johnny Doeie"}
+{"_key": "my.logs", "name": "Johnny Doeie"}
 ```
 
 ### id_key
@@ -359,10 +374,10 @@ id_key request_id # use "request_id" field as a record id in ES
 
 By default, all records inserted into ElasticSearch get a random _id. This option allows to use a field in the record as an identifier.
 
-This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will trigger the following ElasticSearch command
+This following record `{"name": "Johnny", "request_id": "87d89af7daffad6"}` will trigger the following ElasticSearch command
 
 ```
-{ "index" : { "_index" : "logstash-2013.01.01, "_type" : "fluentd", "_id" : "87d89af7daffad6" } }
+{ "index" : { "_index": "logstash-2013.01.01", "_type": "fluentd", "_id": "87d89af7daffad6" } }
 { "name": "Johnny", "request_id": "87d89af7daffad6" }
 ```
 
@@ -380,7 +395,7 @@ If your input is
 ElasticSearch command would be
 
 ```
-{ "index" : { "_index" : "****", "_type" : "****", "_id" : "****", "_parent" : "my_parent" } }
+{ "index" : { "_index": "****", "_type": "****", "_id": "****", "_parent": "my_parent" } }
 { "name": "Johnny", "a_parent": "my_parent" }
 ```
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.0.0.rc.5'
+  s.version       = '2.0.0.rc.6'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -121,6 +121,13 @@ module Fluent::Plugin
       rescue LoadError
         @dump_proc = Yajl.method(:dump)
       end
+
+      if @user && m = @user.match(/%{(?<user>.*)}/)
+        @user = URI.encode_www_form_component(m["user"])
+      end
+      if @password && m = @password.match(/%{(?<password>.*)}/)
+        @password = URI.encode_www_form_component(m["password"])
+      end
     end
 
     def create_meta_config_map
@@ -188,6 +195,18 @@ module Fluent::Plugin
       end
     end
 
+    def get_escaped_userinfo(host_str)
+      if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
+        m["scheme"] +
+          URI.encode_www_form_component(m["user"]) +
+          ':' +
+          URI.encode_www_form_component(m["password"]) +
+          m["path"]
+      else
+        host_str
+      end
+    end
+
     def get_connection_options
       raise "`password` must be present if `user` is present" if @user && !@password
 
@@ -202,7 +221,7 @@ module Fluent::Plugin
             }
           else
             # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
-            uri = URI(host_str)
+            uri = URI(get_escaped_userinfo(host_str))
             %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
               hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
               hash

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -6,6 +6,8 @@ module Fluent::Plugin
 
     Fluent::Plugin.register_output('elasticsearch_dynamic', self)
 
+    helpers :event_emitter
+
     config_param :delimiter, :string, :default => "."
 
     DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
@@ -81,7 +83,7 @@ module Fluent::Plugin
             }
           else
             # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
-            uri = URI(host_str)
+            uri = URI(get_escaped_userinfo(host_str))
             %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
               hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
               hash
@@ -128,17 +130,23 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
-        # evaluate all configurations here
-        DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
-          k = DYNAMIC_PARAM_NAMES[i]
-          v = self.instance_variable_get(var)
-          # check here to determine if we should evaluate
-          if dynamic_conf[k] != v
-            value = expand_param(v, tag, time, record)
-            dynamic_conf[k] = value
-          end
-        }
+        begin
+          # evaluate all configurations here
+          DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
+            k = DYNAMIC_PARAM_NAMES[i]
+            v = self.instance_variable_get(var)
+            # check here to determine if we should evaluate
+            if dynamic_conf[k] != v
+              value = expand_param(v, tag, time, record)
+              dynamic_conf[k] = value
+            end
+          }
         # end eval all configs
+        rescue => e
+          # handle dynamic parameters misconfigurations
+          router.emit_error_event(tag, time, record, e)
+          next
+        end
 
         if eval_or_val(dynamic_conf['logstash_format'])
           if record.has_key?("@timestamp")

data/test/plugin/test_out_elasticsearch.rb

@@ -335,6 +335,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_hosts_list_with_escape_placeholders
+    config = %{
+      hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver(config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'passw%40rd', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -354,6 +380,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal nil, host1[:path]
   end
 
+  def test_single_host_params_and_defaults_with_escape_placeholders
+    config = %{
+      host     logs.google.com
+      user     %{j+hn}
+      password %{d@e}
+    }
+    instance = driver(config).instance
+
+    assert_equal 1, instance.get_connection_options[:hosts].length
+    host1 = instance.get_connection_options[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'd%40e', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -1,8 +1,11 @@
 require 'helper'
 require 'date'
 require 'fluent/test/driver/output'
+require 'flexmock/test_unit'
 
 class ElasticsearchOutputDynamic < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   attr_accessor :index_cmds, :index_command_counts
 
   def setup
@@ -140,6 +143,32 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  def test_hosts_list_with_escape_placeholders
+    config = %{
+      hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver(config).instance
+
+    assert_equal 2, instance.get_connection_options(nil)[:hosts].length
+    host1, host2 = instance.get_connection_options(nil)[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'passw%40rd', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -159,6 +188,25 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal nil, host1[:path]
   end
 
+  def test_single_host_params_and_defaults_with_escape_placeholders
+    config = %{
+      host     logs.google.com
+      user     %{j+hn}
+      password %{d@e}
+    }
+    instance = driver(config).instance
+
+    assert_equal 1, instance.get_connection_options(nil)[:hosts].length
+    host1 = instance.get_connection_options(nil)[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'j%2Bhn', host1[:user]
+    assert_equal 'd%40e', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
@@ -566,6 +614,17 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
   end
 
+  def test_tag_parts_index_error_event
+    stub_elastic_ping
+    stub_elastic
+    driver.configure("logstash_prefix ${tag_parts[1]}\n")
+    flexmock(driver.instance.router).should_receive(:emit_error_event)
+      .with('test', Fluent::EventTime, Hash, TypeError).once
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+  end
+
   def test_connection_failed_retry
     connection_resets = 0
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc.5
+  version: 2.0.0.rc.6
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-25 00:00:00.000000000 Z
+date: 2017-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -170,7 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: ElasticSearch output plugin for Fluent event collector