fluent-plugin-elasticsearch 4.1.1 → 5.4.3

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     @driver = nil
   end
 
-  def driver(conf='', es_version=5)
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def driver(conf='', es_version=elasticsearch_version.to_i)
     # For request stub to detect compatibility.
     @es_version ||= es_version
     Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
@@ -34,6 +42,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }.configure(conf)
   end
 
+  def elasticsearch_transport_layer_decoupling?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+  end
+
+  def elastic_transport_layer?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -51,7 +67,12 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
-    end
+    end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
@@ -71,7 +92,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, url).with do |req|
       index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
       @index_command_counts[url] += index_cmds.size
-    end
+    end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def assert_logs_include(logs, msg, exp_matches=1)
@@ -101,8 +122,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
-                   instance.ssl_version_options)
+      if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      else
+        assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
+                     instance.ssl_version_options)
+      end
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
                    instance.ssl_version_options)
@@ -118,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'configure compression' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
@@ -129,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check compression strategy' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_speed
@@ -140,43 +166,69 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check content-encoding header with compression' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
     }
     instance = driver(config).instance
 
-    assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    if elastic_transport_layer?
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
 
     stub_request(:post, "http://localhost:9200/_bulk").
-      to_return(status: 200, body: "", headers: {})
+      to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     compressable = instance.compressable_connection
 
-    assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    if elastic_transport_layer?
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
   end
 
   test 'check compression option is passed to transport' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
     }
     instance = driver(config).instance
 
-    assert_equal false, instance.client.transport.options[:compression]
+    if elastic_transport_layer?
+      assert_equal false, instance.client.transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal false, instance.client.transport.transport.options[:compression]
+    else
+      assert_equal false, instance.client.transport.options[:compression]
+    end
 
     stub_request(:post, "http://localhost:9200/_bulk").
-      to_return(status: 200, body: "", headers: {})
+      to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     compressable = instance.compressable_connection
 
-    assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    if elastic_transport_layer?
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
+      assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
+    else
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    end
   end
 
   test 'configure Content-Type' do
@@ -335,14 +387,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     if Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-                          with(headers: { "Content-Type" => "application/x-ndjson" })
+                          with(headers: { "Content-Type" => "application/x-ndjson"}).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     else
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-                          with(headers: { "Content-Type" => "application/json" })
+                          with(headers: { "Content-Type" => "application/json"}).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     end
+    stub_elastic_info('http://localhost:9200')
+
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -351,6 +407,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_writes_to_default_index
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -367,7 +424,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_writes_to_default_index_with_compression
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %[
       compression_level default_compression
@@ -389,7 +446,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
 
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-        to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
+        to_return(:status => 200, :headers => {'Content-Type' => 'Application/json', 'x-elastic-product' => 'Elasticsearch'}, :body => compressed_body)
+    stub_elastic_info
 
     driver(config)
     driver.run(default_tag: 'test') do
@@ -401,15 +459,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_writes_to_default_type
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(default_type_name, index_cmds.first['index']['_type'])
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      assert_nil(index_cmds.first['index']['_type'])
+    elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
+      assert_equal("_doc", index_cmds.first['index']['_type'])
+    else
+      assert_equal("fluentd", index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_specified_index
     driver.configure("index_name myindex\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -419,6 +485,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_specified_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -428,15 +495,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_specified_type
     driver.configure("type_name mytype\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('mytype', index_cmds.first['index']['_type'])
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      assert_nil(index_cmds.first['index']['_type'])
+    elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
+      assert_equal("_doc", index_cmds.first['index']['_type'])
+    else
+      assert_equal("mytype", index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_specified_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
+    stub_elastic_info("http://192.168.33.50:9200/")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -446,6 +521,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_specified_port
     driver.configure("port 9201\n")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
+    stub_elastic_info("http://localhost:9201/")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -461,6 +537,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     hosts.each do |host_info|
       host, port = host_info
       stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+      stub_elastic_info("http://#{host}:#{port}/")
     end
 
     driver.run(default_tag: 'test') do
@@ -496,6 +573,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     ]}
 
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
     end
@@ -509,6 +587,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     expected_output = {"foo" => {"bar" => "baz"}}
 
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
     end
@@ -517,6 +596,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_makes_bulk_request
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
       driver.feed(sample_record.merge('age' => 27))
@@ -526,6 +606,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_all_records_are_preserved_in_bulk
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
       driver.feed(sample_record.merge('age' => 27))
@@ -539,6 +620,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -551,6 +633,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -563,6 +646,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -577,6 +661,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -589,18 +674,20 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
-    def test_writes_to_logstash_index_with_specified_dateformat
+  def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
                       logstash_dateformat %Y.%m")
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -614,6 +701,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -622,6 +710,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -631,6 +720,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_adds_logstash_timestamp_when_configured
     driver.configure("logstash_format true\n")
     stub_elastic
+    stub_elastic_info
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
@@ -643,6 +733,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_precision 3\n")
     stub_elastic
+    stub_elastic_info
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
@@ -654,6 +745,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("include_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -665,6 +757,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_uses_custom_timestamp_when_included_in_record_logstash
     driver.configure("logstash_format true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -677,6 +770,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -689,6 +783,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("include_timestamp true
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -702,6 +797,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       index_name test
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -716,6 +812,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key vtm
                       time_key_exclude_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -728,6 +825,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key vtm
                       time_key_exclude_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -737,6 +835,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_doesnt_add_tag_key_by_default
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -746,6 +845,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_adds_tag_key_when_configured
     driver.configure("include_tag_key true\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'mytag') do
       driver.feed(sample_record)
     end
@@ -756,6 +856,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_adds_id_key_when_configured
     driver.configure("id_key request_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -766,6 +867,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_id_key_with_dot
       driver.configure("id_key nested.request_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -775,6 +877,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_id_key_with_dollar_dot
       driver.configure("id_key $.nested.request_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -784,6 +887,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_id_key_with_bracket
       driver.configure("id_key $['nested']['request_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -794,6 +898,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -802,6 +907,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_adds_id_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -811,6 +917,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_adds_parent_key_when_configured
     driver.configure("parent_key parent_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -821,6 +928,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_dot
       driver.configure("parent_key nested.parent_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -830,6 +938,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_dollar_dot
       driver.configure("parent_key $.nested.parent_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -839,6 +948,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_bracket
       driver.configure("parent_key $['nested']['parent_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -849,6 +959,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_doesnt_add_parent_key_if_missing_when_configured
     driver.configure("parent_key another_parent_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -857,6 +968,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_adds_parent_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -867,6 +979,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_es6
       driver("routing_key routing_id\n", 6)
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
@@ -876,6 +989,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_es7
       driver("routing_key routing_id\n", 7)
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
@@ -887,51 +1001,62 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_adds_nested_routing_key_with_dot
       driver.configure("routing_key nested.routing_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
 
     def test_adds_nested_routing_key_with_dollar_dot
       driver.configure("routing_key $.nested.routing_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
 
     def test_adds_nested_routing_key_with_bracket
       driver.configure("routing_key $['nested']['routing_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
   end
 
   def test_doesnt_add_routing_key_if_missing_when_configured
     driver.configure("routing_key another_routing_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert(!index_cmds[0]['index'].has_key?('_routing'))
+    routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+    assert(!index_cmds[0]['index'].has_key?(routing_key))
   end
 
   def test_adds_routing_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert(!index_cmds[0]['index'].has_key?('_routing'))
+    routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+    assert(!index_cmds[0]['index'].has_key?(routing_key))
   end
 
   def test_remove_one_key
     driver.configure("remove_keys key1\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
     end
@@ -942,6 +1067,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_remove_multi_keys
     driver.configure("remove_keys key1, key2\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
     end
@@ -951,6 +1077,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_request_error
     stub_elastic_unavailable
+    stub_elastic_info
     assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
         driver.feed(sample_record)
@@ -962,6 +1089,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     omit("retry_forever test is unstable.") if ENV["CI"]
 
     stub_elastic
+    stub_elastic_info
     driver.configure(Fluent::Config::Element.new(
                'ROOT', '', {
                  '@type' => 'elasticsearch',
@@ -981,6 +1109,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_tag_parts_index_error_event
     stub_elastic
+    stub_elastic_info
     driver.configure("logstash_prefix ${tag_parts[1]}\n")
     flexmock(driver.instance.router).should_receive(:emit_error_event)
       .with('test', Fluent::EventTime, Hash, TypeError).once
@@ -995,7 +1124,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
-    end
+    end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
@@ -1011,7 +1141,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
-    end
+    end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver.configure("reconnect_on_error true\n")
 
@@ -1037,7 +1168,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
       connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
-    end
+    end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver.configure("reconnect_on_error false\n")
 
@@ -1058,6 +1190,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1067,6 +1200,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_upsert_should_not_write_if_theres_no_id
     driver.configure("write_operation upsert\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1076,6 +1210,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_create_should_not_write_if_theres_no_id
     driver.configure("write_operation create\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1086,6 +1221,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("write_operation update
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1097,6 +1233,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1108,6 +1245,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("write_operation create
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -1116,6 +1254,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_include_index_in_url
     stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
+    stub_elastic_info('http://localhost:9200/')
 
     driver.configure("index_name logstash-2018.01.01
                       include_index_in_url true")