fluent-plugin-elasticsearch 3.5.5 → 3.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15eef0c037d870e4d2aed6f8be745c091c2ee2afbb0f43b65f1de5950fd5d2f6
-  data.tar.gz: 46b5d7cfd290ff1a8ef538b508f657a82eab00ec56efdad2367fc3edddff022f
+  metadata.gz: 91343c74b95ee9133fd8323079dbbdd89e539f07942dd936d7107df9dfc5bf8f
+  data.tar.gz: f51a62e66164d542e7aa2d21cfa6565085e3bf6be1db92cc31adb4ad5709588f
 SHA512:
-  metadata.gz: 8b25f7061cf50d0eb7bc9cf0f157de35c528726c356decbd633ad65b58a8494eaeb15c2ab61476d85231e947d85c2e3a738c94cd10ba5d30196882d8c00f5591
-  data.tar.gz: d0a82495ab8ed69725bb9581bf04983db8359fe5c5fbc401e5e8be9b16773942f2f6b58167ed051ec65624ac2daa28344ff1319f97d42d3a8c7c2ae68a944c9d
+  metadata.gz: 283d651e87233d84bd871df3f1d8b472d91f47dc53953ac47b282cfee654762b0199c7779a398485c24eb89885a01aff5204481a660ed6d7e0fb5243ccbfb2ef
+  data.tar.gz: 6ae7ab7dac0721dd82082e01ca9de3bc969949159d9b263cf53e43d62d01a543cb84ba6992d46189f16ab4b9b1c63de2ac8dd49b45e9778e7a9ec01fa66395c8

data/History.md

@@ -1,6 +1,13 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 3.5.6
+- Support elasticsearch8 removal of mapping types (#656)
+- Upgrade webmock to 3 (#652)
+- Suppress `ruby -c` warnings (#649)
+- Add tips for sniffer class (#644)
+- Make `client_key_pass` secret (#637)
+- Validate `user` and `password` early (#636)
 
 ### 3.5.5
 - Fix arguments order of `assert_equal` (#635)

data/README.md

@@ -132,6 +132,10 @@ In your Fluentd configuration, use `@type elasticsearch`. Additional configurati
 </match>
 ```
 
+NOTE: `type_name` parameter will be used fixed `_doc` value for Elasticsearch 7.
+
+NOTE: `type_name` parameter will make no effect for Elasticsearch 8.
+
 ### Index templates
 
 This plugin creates Elasticsearch indices by merely writing to them. Consider using [Index Templates](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) to gain control of what get indexed and how. See [this example](https://github.com/uken/fluent-plugin-elasticsearch/issues/33#issuecomment-38693282) for a good starting point.
@@ -894,6 +898,25 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
 reload_after 100
 ```
 
+#### Tips
+
+The included sniffer class does not required `out_elasticsearch`.
+You should tell Fluentd where the sniffer class exists.
+
+If you use td-agent, you must put the following lines into `TD_AGENT_DEFAULT` file:
+
+```
+sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
+TD_AGENT_OPTIONS="--use-v1-config -r $sniffer"
+```
+
+If you use Fluentd directly, you must pass the following lines as Fluentd command line option:
+
+```
+sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
+$ fluentd -r $sniffer" [AND YOUR OTHER OPTIONS]
+```
+
 ### Reload After
 
 When `reload_connections true`, this is the integer number of operations after which the plugin will

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.5.5'
+  s.version       = '3.5.6'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
 
 
   s.add_development_dependency 'rake', '>= 0'
-  s.add_development_dependency 'webmock', '~> 1'
+  s.add_development_dependency 'webmock', '~> 3'
   s.add_development_dependency 'test-unit', '~> 3.1.0'
   s.add_development_dependency 'minitest', '~> 5.8'
   s.add_development_dependency 'flexmock', '~> 2.0'

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -27,7 +27,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     if @plugin.log_es_400_reason
       block.call
     else
-      @plugin.log.on_debug &block
+      @plugin.log.on_debug(&block)
     end
   end
 

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -57,7 +57,7 @@ module Fluent::ElasticsearchIndexTemplate
 
   def indexcreation(index_name)
     client.indices.create(:index => index_name)
-    rescue Elasticsearch::Transport::Transport::Error => e
+  rescue Elasticsearch::Transport::Transport::Error => e
       log.error("Error while index creation - #{index_name}: #{e.inspect}")
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -91,7 +91,7 @@ EOC
     config_param :ssl_verify , :bool, :default => true
     config_param :client_key, :string, :default => nil
     config_param :client_cert, :string, :default => nil
-    config_param :client_key_pass, :string, :default => nil
+    config_param :client_key_pass, :string, :default => nil, :secret => true
     config_param :ca_file, :string, :default => nil
     config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1
     config_param :remove_keys, :string, :default => nil
@@ -218,6 +218,8 @@ EOC
         @dump_proc = Yajl.method(:dump)
       end
 
+      raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
+
       if @user && m = @user.match(/%{(?<user>.*)}/)
         @user = URI.encode_www_form_component(m["user"])
       end
@@ -251,10 +253,14 @@ EOC
       if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
         log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
       end
-      if @last_seen_major_version >= 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+      if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+        log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
         @type_name = '_doc'.freeze
       end
+      if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+        log.info "Detected ES 8.x or above: This parameter has no effect."
+        @type_name = nil
+      end
 
       if @validate_client_version && !Fluent::Engine.dry_run_mode
         if @last_seen_major_version != client_library_version.to_i
@@ -454,7 +460,6 @@ EOC
     end
 
     def get_connection_options(con_host=nil)
-      raise "`password` must be present if `user` is present" if @user && !@password
 
       hosts = if con_host || @hosts
         (con_host || @hosts).split(',').map do |host_str|
@@ -569,7 +574,11 @@ EOC
     def expand_placeholders(chunk)
       logstash_prefix = extract_placeholders(@logstash_prefix, chunk)
       index_name = extract_placeholders(@index_name, chunk)
-      type_name = extract_placeholders(@type_name, chunk)
+      if @type_name
+        type_name = extract_placeholders(@type_name, chunk)
+      else
+        type_name = nil
+      end
       return logstash_prefix, index_name, type_name
     end
 
@@ -688,14 +697,20 @@ EOC
         if @last_seen_major_version == 6
           log.warn "Detected ES 6.x: `@type_name` will be used as the document `_type`."
           target_type = type_name
-        elsif @last_seen_major_version >= 7
-          log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+        elsif @last_seen_major_version == 7
+          log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
+        elsif @last_seen_major_version >=8
+          log.warn "Detected ES 8.x or above: document type will not be used."
+          target_type = nil
         end
       else
-        if @last_seen_major_version >= 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-          log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
+        elsif @last_seen_major_version >= 8
+          log.warn "Detected ES 8.x or above: document type will not be used."
+          target_type = nil
         else
           target_type = type_name
         end
@@ -703,7 +718,7 @@ EOC
 
       meta.clear
       meta["_index".freeze] = target_index
-      meta["_type".freeze] = target_type
+      meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
 
       if @pipeline
         meta["pipeline".freeze] = @pipeline

data/test/plugin/test_out_elasticsearch.rb

@@ -277,6 +277,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '_doc', instance.type_name
   end
 
+  test 'Detected Elasticsearch 8' do
+    config = %{
+      type_name noeffect
+    }
+    instance = driver(config, 8).instance
+    assert_equal nil, instance.type_name
+  end
+
   test 'Detected Elasticsearch 6 and insecure security' do
     config = %{
       ssl_version TLSv1_1
@@ -329,7 +337,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                  'scheme' => 'https',
                  'path' => '/es/',
                  'user' => 'john',
-                 'pasword' => 'doe',
+                 'password' => 'doe',
                }, [
                  Fluent::Config::Element.new('buffer', 'mykey', {
                                                'chunk_keys' => 'mykey'
@@ -349,7 +357,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                  'scheme' => 'https',
                  'path' => '/es/',
                  'user' => 'john',
-                 'pasword' => 'doe',
+                 'password' => 'doe',
                }, [
                  Fluent::Config::Element.new('buffer', 'tag', {
                                              }, [])
@@ -392,7 +400,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       }
 
       connection_resets = 0
-      stub_request(:get, "https://john:doe@logs.google.com:778/es//").with do |req|
+      stub_request(:get, "https://logs.google.com:778/es//").
+        with(basic_auth: ['john', 'doe']) do |req|
         connection_resets += 1
         raise Faraday::ConnectionFailed, "Test message"
       end
@@ -418,15 +427,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
+    assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash")
   end
 
   def test_template_create
@@ -445,18 +456,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
   end
 
   def test_custom_template_create
@@ -476,18 +490,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
   def test_custom_template_with_rollover_index_create
@@ -512,27 +529,33 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # creation of index which can rollover
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp_deflector").
+    stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
 
@@ -553,18 +576,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
   end
 
   def test_custom_template_overwrite
@@ -585,18 +611,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
   def test_custom_template_with_rollover_index_overwrite
@@ -621,27 +650,33 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # creation of index which can rollover
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp_deflector").
+    stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
   def test_template_create_invalid_filename
@@ -657,10 +692,12 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
 
     assert_raise(RuntimeError) {
@@ -706,7 +743,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     connection_resets = 0
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
+    stub_request(:get, "https://logs.google.com:778/es//_template/logstash")
+      .with(basic_auth: ['john', 'doe']) do |req|
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
@@ -737,7 +775,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     connection_resets = 0
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
+    stub_request(:get, "https://logs.google.com:778/es//_template/logstash")
+      .with(basic_auth: ['john', 'doe']) do |req|
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
@@ -760,29 +799,36 @@ class ElasticsearchOutput < Test::Unit::TestCase
       templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
     }
 
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
      # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
 
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash3").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {}) #exists
 
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash3").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
-    assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
+    assert_requested( :put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_requested( :put, "https://logs.google.com:777/es//_template/logstash2", times: 1)
+    assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash3") #exists
   end
 
   def test_templates_overwrite
@@ -799,28 +845,35 @@ class ElasticsearchOutput < Test::Unit::TestCase
       template_overwrite true
     }
 
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
      # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash3").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {}) #exists
 
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash3").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash2", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash3", times: 1)
   end
 
   def test_templates_not_used
@@ -839,29 +892,36 @@ class ElasticsearchOutput < Test::Unit::TestCase
       templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
     }
     # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
     #creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
 
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
+    assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash1")
+    assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash2")
   end
 
   def test_templates_can_be_partially_created_if_error_occurs
@@ -876,25 +936,30 @@ class ElasticsearchOutput < Test::Unit::TestCase
       password        doe
       templates       {"logstash1":"#{template_file}", "logstash2":"/abc" }
     }
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
      # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
 
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
+    stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
+      with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
 
     assert_raise(RuntimeError) {
       driver(config)
     }
 
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
+    assert_requested(:put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
+    assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash2")
   end
 
   def test_legacy_hosts_list
@@ -1024,6 +1089,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert(ports.none? { |p| p == 9200 })
   end
 
+  def test_password_is_required_if_specify_user
+    config = %{
+      user john
+    }
+
+    assert_raise(Fluent::ConfigError) do
+      driver(config)
+    end
+  end
+
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
@@ -1063,20 +1138,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
   end
 
-  def test_writes_to_default_index
+  data('Elasticsearch 6' => [6, 'fluentd'],
+       'Elasticsearch 7' => [7, 'fluentd'],
+       'Elasticsearch 8' => [8, 'fluentd'],
+      )
+  def test_writes_to_default_index(data)
+    version, index_name = data
     stub_elastic
+    driver("", version)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('fluentd', index_cmds.first['index']['_index'])
+    assert_equal(index_name, index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_default_type
+  data('Elasticsearch 6' => [6, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME],
+       'Elasticsearch 7' => [7, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME_ES_7x],
+       'Elasticsearch 8' => [8, nil],
+      )
+  def test_writes_to_default_type(data)
+    version, index_type = data
     stub_elastic
+    driver("", version)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(default_type_name, index_cmds.first['index']['_type'])
+    assert_equal(index_type, index_cmds.first['index']['_type'])
   end
 
   def test_writes_to_speficied_index

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -141,7 +141,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                  'scheme' => 'https',
                  'path' => '/es/',
                  'user' => 'john',
-                 'pasword' => 'doe',
+                 'password' => 'doe',
                }, [
                  Fluent::Config::Element.new('buffer', 'tag', {
                                              }, [])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.5.5
+  version: 3.5.6
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-26 00:00:00.000000000 Z
+date: 2019-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -73,14 +73,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement