fluent-plugin-elasticsearch 5.0.2 → 5.1.0

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -27,13 +27,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
+        raise 'process_message'
       end
       return true
     end
@@ -302,7 +307,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   def test_retry_error
     records = []
     error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
+    error_records.merge!({0=>true, 4=>true})
     10.times do |i|
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
@@ -386,6 +391,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
               "reason":"unrecognized error"
             }
           }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "9",
+            "status" : 500,
+            "error" : {
+              "type" : "json_parse_exception",
+              "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
+            }
+          }
         }
       ]
     }))
@@ -400,12 +417,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 2, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 8, records[1]['_id']
+      assert_equal 2, records.length, "Exp. retry_stream to contain records"
+      assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
+      assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 3, error_ids.length
-      assert_equal [5, 6, 7], error_ids
+      assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
+      assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -19,7 +19,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   end
 
   def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
   end
 

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -27,9 +27,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     assert_equal(expected, xpack_info)
   end
@@ -37,18 +43,21 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
       to_return(:status => 404, :body => "", :headers => {})
+    stub_elastic_info
     assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
   end
 
@@ -59,6 +68,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     create_ilm_policy("fluent-policy")
 
     assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)

data/test/plugin/test_in_elasticsearch.rb

@@ -31,6 +31,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def sample_response(index_name="fluentd")
     {
       "took"=>4,
@@ -322,6 +327,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG)
     driver.run(expect_emits: 1, timeout: 10)
@@ -337,6 +343,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
     driver.run(expect_emits: 1, timeout: 10)
@@ -352,6 +359,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
     driver.run(expect_emits: 1, timeout: 10)
@@ -370,6 +378,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
                       timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
@@ -389,6 +398,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
     driver.run(expect_emits: 1, timeout: 10)
@@ -412,6 +422,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
     driver.run(expect_emits: 1, timeout: 10)
@@ -434,6 +445,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1
     end
+    stub_elastic_info
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,

data/test/plugin/test_out_elasticsearch.rb

@@ -10,7 +10,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
-  attr_accessor :index_cmds, :index_command_counts
+  attr_accessor :index_cmds, :index_command_counts, :index_cmds_all_requests
 
   def setup
     Fluent::Test.setup
@@ -45,6 +45,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }.configure(conf)
   end
 
+  def elasticsearch_transport_layer_decoupling?
+    Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -60,7 +64,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   end
 
   def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
   end
 
@@ -70,6 +74,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  def stub_elastic_all_requests(url="http://localhost:9200/_bulk")
+    @index_cmds_all_requests = Array.new
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_cmds_all_requests << @index_cmds
+    end
+  end
+
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
@@ -253,7 +265,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_true instance.verify_es_version_at_startup
     assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
     assert_false instance.log_es_400_reason
-    assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
+    assert_equal -1, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TARGET_BULK_BYTES
     assert_false instance.compression
     assert_equal :no_compression, instance.compression_level
     assert_true instance.http_backend_excon_nonblock
@@ -290,16 +302,25 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    if elasticsearch_transport_layer_decoupling?
+      assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
 
     stub_request(:post, "http://localhost:9200/_bulk").
       to_return(status: 200, body: "", headers: {})
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     compressable = instance.compressable_connection
 
-    assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    if elasticsearch_transport_layer_decoupling?
+      assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
+    else
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    end
   end
 
   test 'check compression option is passed to transport' do
@@ -310,16 +331,25 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal false, instance.client.transport.options[:compression]
+    if elasticsearch_transport_layer_decoupling?
+      assert_equal false, instance.client.transport.transport.options[:compression]
+    else
+      assert_equal false, instance.client.transport.options[:compression]
+    end
 
     stub_request(:post, "http://localhost:9200/_bulk").
       to_return(status: 200, body: "", headers: {})
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     compressable = instance.compressable_connection
 
-    assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    if elasticsearch_transport_layer_decoupling?
+      assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
+    else
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    end
   end
 
   test 'check configure cloud_id based client' do
@@ -406,7 +436,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
 
     def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
-      body ="{\"version\":{\"number\":\"#{version}\"}}"
+      body ="{\"version\":{\"number\":\"#{version}\",\"build_flavor\":\"default\"},\"tagline\":\"You Know, for Search\"}"
       stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
     end
 
@@ -423,9 +453,15 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         scheme https
         @log_level info
       }
-      d = create_driver(config, 7, "\"7.10.1\"")
-      logs = d.logs
-      assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
+      if elasticsearch_transport_layer_decoupling?
+        assert_raise(NoMethodError) do
+          d = create_driver(config, 7, "\"7.10.1\"")
+        end
+      else
+        d = create_driver(config, 7, "\"7.10.1\"")
+        logs = d.logs
+        assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
+      end
     end
   end
 
@@ -488,6 +524,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
         with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(status: 200, body: "", headers: {})
+      stub_elastic_info
 
       assert_nothing_raised {
         driver(config)
@@ -532,6 +569,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
         with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"75gb\",\"max_age\":\"50d\"}}}}}}").
         to_return(status: 200, body: "", headers: {})
+      stub_elastic_info
 
       assert_nothing_raised {
         driver(config)
@@ -548,6 +586,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         template_file #{template_file}
         ilm_policy_overwrite true
       }
+      stub_elastic_info
 
       assert_raise(Fluent::ConfigError) {
         driver(config)
@@ -930,6 +969,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -974,6 +1014,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -1037,6 +1078,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver(config)
 
     elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+    stub_elastic_info("https://logs.google.com:777/es//")
     driver.run(default_tag: 'test.template') do
       driver.feed(sample_record)
     end
@@ -1102,6 +1144,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe'],
            body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -1212,6 +1255,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1298,6 +1342,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1389,6 +1434,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1497,6 +1543,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1581,6 +1628,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1665,6 +1713,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -1756,6 +1805,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
           with(basic_auth: ['john', 'doe'],
                body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
           to_return(:status => 200, :body => "", :headers => {})
+        stub_elastic_info("https://logs.google.com:777/es//")
 
         driver(config)
 
@@ -1846,6 +1896,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
           with(basic_auth: ['john', 'doe'],
                body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
           to_return(:status => 200, :body => "", :headers => {})
+        stub_elastic_info("https://logs.google.com:777/es//")
 
         driver(config)
 
@@ -1971,6 +2022,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -2067,6 +2119,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -2163,6 +2216,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -2218,6 +2272,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -2270,6 +2325,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver(config)
 
     stub_elastic("https://logs.google.com:777/es//_bulk")
+    stub_elastic_info("https://logs.google.com:777/es//")
     driver.run(default_tag: 'test.template') do
       driver.feed(sample_record)
     end
@@ -2321,6 +2377,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver(config)
 
     stub_elastic("https://logs-test.google.com:777/es//_bulk")
+    stub_elastic_info("https://logs-test.google.com:777/es//")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -2381,6 +2438,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -2443,6 +2501,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -2511,6 +2570,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver(config)
 
     elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+    stub_elastic_info("https://logs.google.com:777/es//")
     driver.run(default_tag: 'custom-test') do
       driver.feed(sample_record)
     end
@@ -2610,7 +2670,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
-
+      stub_elastic_info("https://logs.google.com:777/es//")
       driver(config)
 
       assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
@@ -2699,6 +2759,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -2728,6 +2789,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       }
 
       # Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
+      stub_elastic_info("https://logs.google.com:777/es//")
       assert_raise(Fluent::ConfigError) do
         driver(config)
       end
@@ -2863,6 +2925,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
+      stub_elastic_info("https://logs.google.com:777/es//")
 
       driver(config)
 
@@ -2953,7 +3016,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe'],
              :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
-
+      stub_elastic_info("https://logs.google.com:777/es//")
       driver(config)
 
       assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
@@ -2999,6 +3062,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3045,6 +3109,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3107,6 +3172,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3133,7 +3199,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
-
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     assert_raise(RuntimeError) {
       driver(config)
@@ -3185,7 +3251,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
+    stub_elastic("https://logs-test.google.com:777/es//_bulk")
+    stub_elastic_info("https://logs-test.google.com:777/es//")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -3225,6 +3292,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
+    stub_elastic_info("https://logs.google.com:778/es//")
 
     assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
       driver(config)
@@ -3266,6 +3334,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       retries += 1
       raise error
     end
+    stub_elastic_info("https://logs.google.com:778/es//")
 
     assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
       driver(config)
@@ -3309,6 +3378,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
+    stub_elastic_info("https://logs.google.com:778/es//")
 
     driver(config)
 
@@ -3364,6 +3434,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3421,6 +3492,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3479,6 +3551,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     driver(config)
 
@@ -3529,6 +3602,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info("https://logs.google.com:777/es//")
 
     assert_raise(RuntimeError) {
       driver(config)
@@ -3545,6 +3619,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       path     /es/
       port     123
     }
+    stub_elastic_info("https://host1:50")
+    stub_elastic_info("https://host2:100")
+    stub_elastic_info("https://host3:123")
     instance = driver(config).instance
 
     assert_equal 3, instance.get_connection_options[:hosts].length
@@ -3567,6 +3644,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       user     default_user
       password default_password
     }
+    stub_elastic_info("https://john:password@host1:443/elastic/")
+    stub_elastic_info("http://host2")
     instance = driver(config).instance
 
     assert_equal 2, instance.get_connection_options[:hosts].length
@@ -3593,6 +3672,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       user     default_user
       password default_password
     }
+    stub_elastic_info("https://j%2Bhn:passw%40rd@host1:443/elastic/")
+    stub_elastic_info("http://host2")
+
     instance = driver(config).instance
 
     assert_equal 2, instance.get_connection_options[:hosts].length
@@ -3612,6 +3694,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
+  class IPv6AdressStringHostsTest < self
+    def test_legacy_hosts_list
+      config = %{
+        hosts    "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
+        scheme   https
+        path     /es/
+        port     123
+      }
+      instance = driver(config).instance
+
+      assert_raise(URI::InvalidURIError) do
+        instance.get_connection_options[:hosts].length
+      end
+    end
+
+    def test_hosts_list
+      config = %{
+        hosts    https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'password', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+
+    def test_hosts_list_with_escape_placeholders
+      config = %{
+        hosts    https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
+        path     /default_path
+        user     default_user
+        password default_password
+      }
+      instance = driver(config).instance
+
+      assert_equal 2, instance.get_connection_options[:hosts].length
+      host1, host2 = instance.get_connection_options[:hosts]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 443, host1[:port]
+      assert_equal 'https', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'passw%40rd', host1[:password]
+      assert_equal '/elastic/', host1[:path]
+
+      assert_equal 'host2', host2[:host]
+      assert_equal 'http', host2[:scheme]
+      assert_equal 'default_user', host2[:user]
+      assert_equal 'default_password', host2[:password]
+      assert_equal '/default_path', host2[:path]
+    end
+  end
+
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
@@ -3665,6 +3815,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert(ports.none? { |p| p == 9200 })
   end
 
+  class IPv6AdressStringHostTest < self
+    def test_single_host_params_and_defaults
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     john
+        password doe
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'john', host1[:user]
+      assert_equal 'doe', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+
+    def test_single_host_params_and_defaults_with_escape_placeholders
+      config = %{
+        host     2404:7a80:d440:3000:192a:a292:bd7f:ca19
+        user     %{j+hn}
+        password %{d@e}
+      }
+      instance = driver(config).instance
+
+      assert_equal 1, instance.get_connection_options[:hosts].length
+      host1 = instance.get_connection_options[:hosts][0]
+
+      assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
+      assert_equal 9200, host1[:port]
+      assert_equal 'http', host1[:scheme]
+      assert_equal 'j%2Bhn', host1[:user]
+      assert_equal 'd%40e', host1[:password]
+      assert_equal nil, host1[:path]
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john
@@ -3685,6 +3875,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/json" })
     end
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -3696,6 +3887,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                         with(headers: {'custom' => 'header1','and_others' => 'header2' })
+    stub_elastic_info
     driver.configure(%[custom_headers {"custom":"header1", "and_others":"header2"}])
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -3708,6 +3900,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                         with(headers: {'Authorization'=>'ApiKey dGVzdGF1dGhoZWFkZXI='})
+    stub_elastic_info
     driver.configure(%[api_key testauthheader])
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -3718,6 +3911,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_write_message_with_bad_chunk
     driver.configure("target_index_key bad_value\n@log_level debug\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed({'bad_value'=>"\255"})
     end
@@ -3733,6 +3927,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_default_index(data)
     version, index_name = data
     stub_elastic
+    stub_elastic_info
     driver("", version)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -3774,6 +3969,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
     elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
         to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
+    stub_elastic_info("http://localhost:9200/")
 
     driver(config)
     driver.run(default_tag: 'test') do
@@ -3790,6 +3986,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_default_type(data)
     version, index_type = data
     stub_elastic
+    stub_elastic_info
     driver("", version)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -3800,6 +3997,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_index
     driver.configure("index_name myindex\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -3819,6 +4017,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                        ]
                      ))
     request = stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
       driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
@@ -3843,6 +4042,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         body: /{"index":{"_index":"fluentd","_type":"fluentd"}}\n{"age":26,"request_id":"42","parent_id":"parent","routing_id":"routing","#{chunk_id_key}":".*"}\n/) do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
+    stub_elastic_info
     driver.run(default_tag: 'test', shutdown: false) do
       driver.feed(sample_record)
     end
@@ -3872,6 +4072,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                        ]
                      ))
     request = stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
       driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
@@ -3884,6 +4085,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_writes_to_speficied_index_with_tag_placeholder
       driver.configure("index_name myindex.${tag}\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
@@ -3903,6 +4105,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                          ]
                        ))
       stub_elastic
+      stub_elastic_info
       time = Time.parse Date.today.iso8601
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
@@ -3923,6 +4126,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       pipeline_id = "mypipeline"
       logstash_index = "myindex.#{pipeline_id}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
       end
@@ -3933,6 +4137,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -3944,6 +4149,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_index_key
     driver.configure("target_index_key @target_index\n")
     stub_elastic
+    stub_elastic_info
     record = sample_record.clone
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('@target_index' => 'local-override'))
@@ -3957,17 +4163,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       logstash_format true")
     time = Time.parse Date.today.iso8601
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
     end
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
 
-   def test_writes_to_target_index_key_logstash_uppercase
+  def test_writes_to_target_index_key_logstash_uppercase
     driver.configure("target_index_key @target_index
                       logstash_format true")
     time = Time.parse Date.today.iso8601
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record.merge('@target_index' => 'LOCAL-OVERRIDE'))
     end
@@ -3980,17 +4188,203 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     pipeline = "fluentd"
     driver.configure("pipeline #{pipeline}")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     assert_equal(pipeline, index_cmds.first['index']['pipeline'])
   end
 
+  def stub_elastic_affinity_target_index_search_with_body(url="http://localhost:9200/logstash-*/_search", ids, return_body_str)
+    # Note: ids used in query is unique list of ids
+    stub_request(:post, url)
+      .with(
+        body: "{\"query\":{\"ids\":{\"values\":#{ids.uniq.to_json}}},\"_source\":false,\"sort\":[{\"_index\":{\"order\":\"desc\"}}]}",
+      )
+      .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => return_body_str
+      }
+    end)
+  end
+
+  def stub_elastic_affinity_target_index_search(url="http://localhost:9200/logstash-*/_search", ids, indices)
+    # Example ids and indices arrays.
+    #  [ "3408a2c8eecd4fbfb82e45012b54fa82", "2816fc6ef4524b3f8f7e869002005433"]
+    #  [ "logstash-2021.04.28", "logstash-2021.04.29"]
+    body = %({
+      "took" : 31,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 52,
+        "successful" : 52,
+        "skipped" : 48,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 356,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [
+          {
+            "_index" : "#{indices[0]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[0]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[0]}"
+            ]
+          },
+          {
+            "_index" : "#{indices[1]}",
+            "_type" : "_doc",
+            "_id" : "#{ids[1]}",
+            "_score" : null,
+            "sort" : [
+              "#{indices[1]}"
+            ]
+          }
+        ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, body)
+  end
+
+  def stub_elastic_affinity_target_index_search_return_empty(url="http://localhost:9200/logstash-*/_search", ids)
+    empty_body = %({
+      "took" : 5,
+      "timed_out" : false,
+      "_shards" : {
+        "total" : 54,
+        "successful" : 54,
+        "skipped" : 53,
+        "failed" : 0
+      },
+      "hits" : {
+        "total" : {
+          "value" : 0,
+          "relation" : "eq"
+        },
+        "max_score" : null,
+        "hits" : [ ]
+      }
+    })
+    stub_elastic_affinity_target_index_search_with_body(ids, empty_body)
+  end
+
+  def test_writes_to_affinity_target_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_info
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_write_operation_upsert
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation upsert")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    indices = ["logstash-2021.04.28"]
+    stub_elastic
+    stub_elastic_info
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_index_not_exists_yet
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value]
+    stub_elastic
+    stub_elastic_info
+    stub_elastic_affinity_target_index_search_return_empty(ids)
+    time = Time.parse Date.today.iso8601
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record('my_id' => my_id_value))
+    end
+    assert_equal("logstash-#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['update']['_index'])
+  end
+
+  def test_writes_to_affinity_target_index_multiple_indices
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    my_id_value2 = "3408a2c8eecd4fbfb82e45012b54fa82"
+    ids = [my_id_value, my_id_value2]
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+    stub_elastic_info
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value2))
+    end
+    assert_equal(2, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.29', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[1].first['update']['_index'])
+    assert_equal(my_id_value2, index_cmds_all_requests[1].first['update']['_id'])
+  end
+
+  def test_writes_to_affinity_target_index_same_id_dublicated_write_to_oldest_index
+    driver.configure("target_index_affinity true
+                      logstash_format true
+                      id_key my_id
+                      write_operation update")
+
+    my_id_value = "2816fc6ef4524b3f8f7e869002005433"
+    # It may happen than same id has inserted to two index while data inserted during rollover period
+    ids = [my_id_value, my_id_value]
+    # Simulate the used sorting here, as search sorts indices in DESC order to pick only oldest index per single _id
+    indices = ["logstash-2021.04.29", "logstash-2021.04.28"]
+
+    stub_elastic_info
+    stub_elastic_all_requests
+    stub_elastic_affinity_target_index_search(ids, indices)
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('my_id' => my_id_value))
+      driver.feed(sample_record('my_id' => my_id_value))
+    end
+    assert_equal('logstash-2021.04.28', index_cmds.first['update']['_index'])
+
+    assert_equal(1, index_cmds_all_requests.count)
+    assert_equal('logstash-2021.04.28', index_cmds_all_requests[0].first['update']['_index'])
+    assert_equal(my_id_value, index_cmds_all_requests[0].first['update']['_id'])
+  end
+
   class PipelinePlaceholdersTest < self
     def test_writes_to_default_index_with_pipeline_tag_placeholder
       pipeline = "fluentd-${tag}"
       driver.configure("pipeline #{pipeline}")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test.builtin.placeholder') do
         driver.feed(sample_record)
       end
@@ -4012,6 +4406,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       pipeline = "fluentd-#{time.getutc.strftime("%Y%m%d")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
       end
@@ -4031,6 +4426,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       pipeline_id = "mypipeline"
       logstash_index = "fluentd-#{pipeline_id}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
       end
@@ -4041,6 +4437,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_index_key_fallack
     driver.configure("target_index_key @target_index\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4053,6 +4450,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4066,6 +4464,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_type(data)
     driver('', data["es_version"]).configure("type_name mytype\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4079,6 +4478,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_type_with_placeholders(data)
     driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4093,6 +4493,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver('', data["es_version"])
       .configure("type_name mytype.${tag}\nsuppress_type_name true")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4107,6 +4508,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_type_key(data)
     driver('', data["es_version"]).configure("target_type_key @target_type\n")
     stub_elastic
+    stub_elastic_info
     record = sample_record.clone
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('@target_type' => 'local-override'))
@@ -4118,6 +4520,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_type_key_fallack_to_default
     driver.configure("target_type_key @target_type\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4128,6 +4531,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("target_type_key @target_type
                       type_name mytype")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4142,6 +4546,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_type_key_nested(data)
     driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('kubernetes' => {
         'labels' => {
@@ -4156,6 +4561,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_target_type_key_fallack_to_default_nested
     driver.configure("target_type_key kubernetes.labels.log_type\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('kubernetes' => {
         'labels' => {
@@ -4169,6 +4575,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
+    stub_elastic_info("http://192.168.33.50:9200/")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4178,6 +4585,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_writes_to_speficied_port
     driver.configure("port 9201\n")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
+    stub_elastic_info("http://localhost:9201")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4193,6 +4601,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     hosts.each do |host_info|
       host, port = host_info
       stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+      stub_elastic_info("http://#{host}:#{port}/")
     end
 
     driver.run(default_tag: 'test') do
@@ -4229,6 +4638,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     ]}
 
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
     end
@@ -4242,6 +4652,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     expected_output = {"foo" => {"bar" => "baz"}}
 
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
     end
@@ -4250,6 +4661,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_makes_bulk_request
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
       driver.feed(sample_record.merge('age' => 27))
@@ -4257,8 +4669,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(4, index_cmds.count)
   end
 
-  def test_all_records_are_preserved_in_bulk
+  def test_all_re
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
       driver.feed(sample_record.merge('age' => 27))
@@ -4274,6 +4687,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
     logstash_index = "logstash-2015.05.31"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(dt.to_time.to_i, sample_record)
     end
@@ -4288,6 +4702,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Date.today.to_time
     index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4300,6 +4715,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4314,6 +4730,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4327,6 +4744,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
       end
@@ -4349,6 +4767,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
       end
@@ -4369,6 +4788,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       pipeline_id = "mypipeline"
       logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
       end
@@ -4394,6 +4814,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m.%d'))
       end
@@ -4417,6 +4838,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m'))
       end
@@ -4429,6 +4851,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       driver.configure("host ${tag}\n")
       time = Time.parse Date.today.iso8601
       elastic_request = stub_elastic("http://extracted-host:9200/_bulk")
+      stub_elastic_info("http://extracted-host:9200/")
       driver.run(default_tag: 'extracted-host') do
         driver.feed(time.to_i, sample_record)
       end
@@ -4445,6 +4868,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         host, port = host_info
         host = "extracted-host" if host == '${tag}'
         stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+        stub_elastic_info("http://#{host}:#{port}")
       end
 
       driver.run(default_tag: 'extracted-host') do
@@ -4479,8 +4903,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                          ]
                        ))
       stub_elastic
+      stub_elastic_info
       time = Time.parse Date.today.iso8601
       elastic_request = stub_elastic("http://host-#{time.utc.strftime('%Y%m%d')}:9200/_bulk")
+      stub_elastic_info("http://host-#{time.utc.strftime('%Y%m%d')}:9200/")
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
       end
@@ -4501,6 +4927,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       second_pipeline_id = "2"
       first_request = stub_elastic("http://myhost-1:9200/_bulk")
       second_request = stub_elastic("http://myhost-2:9200/_bulk")
+      stub_elastic_info("http://myhost-1:9200/")
+      stub_elastic_info("http://myhost-2:9200/")
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => first_pipeline_id}))
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => second_pipeline_id}))
@@ -4521,6 +4949,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       pipeline_id = "1"
       request = stub_elastic_unavailable("http://myhost-1:9200/_bulk")
+      stub_elastic_info("http://myhost-1:9200/")
       exception = assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
         driver.run(default_tag: 'test') do
           driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
@@ -4536,6 +4965,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4550,6 +4980,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4563,6 +4994,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
     end
@@ -4589,6 +5021,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4598,6 +5031,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_adds_timestamp_when_logstash
     driver.configure("logstash_format true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.now
     time = Fluent::EventTime.from_time(ts.to_time)
     driver.run(default_tag: 'test') do
@@ -4610,6 +5044,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_adds_timestamp_when_include_timestamp
     driver.configure("include_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.now
     time = Fluent::EventTime.from_time(ts.to_time)
     driver.run(default_tag: 'test') do
@@ -4622,6 +5057,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -4633,6 +5069,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_uses_custom_timestamp_when_included_in_record_without_logstash
     driver.configure("include_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -4645,6 +5082,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601(9)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -4658,6 +5096,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       time_precision 3
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     time = Time.now
     float_time = time.to_f
     driver.run(default_tag: 'test') do
@@ -4672,6 +5111,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = "2001-02-03 13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -4686,6 +5126,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = "2001-02-03 13:14:01.673+02:00"
     time = Time.parse(ts)
     current_zone_offset = Time.new(2001, 02, 03).to_datetime.offset
@@ -4703,6 +5144,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
                       time_key vtm\n")
     stub_elastic
+    stub_elastic_info
     ts = "2001-02-03 13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -4717,6 +5159,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       time_key vtm
                       time_key_exclude_timestamp true\n")
     stub_elastic
+    stub_elastic_info
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
@@ -4728,6 +5171,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
     stub_elastic
+    stub_elastic_info
     ts = "2001-02-03T13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -4742,6 +5186,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       index_name test
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
     stub_elastic
+    stub_elastic_info
     ts = "2001-02-03T13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -4759,6 +5204,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
     stub_elastic
+    stub_elastic_info
 
     ts = "2001/02/03 13:14:01,673+02:00"
     index = "logstash-#{Time.now.getutc.strftime("%Y.%m.%d")}"
@@ -4779,6 +5225,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key_format %a %b %d %H:%M:%S %Z %Y\n")
     stub_elastic
+    stub_elastic_info
     ts = "Thu Nov 29 14:33:20 GMT 2001"
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -4791,6 +5238,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_uses_nanosecond_precision_by_default
     driver.configure("logstash_format true\n")
     stub_elastic
+    stub_elastic_info
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
@@ -4803,6 +5251,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_precision 3\n")
     stub_elastic
+    stub_elastic_info
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
@@ -4813,6 +5262,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_doesnt_add_tag_key_by_default
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4822,6 +5272,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_adds_tag_key_when_configured
     driver.configure("include_tag_key true\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'mytag') do
       driver.feed(sample_record)
     end
@@ -4832,6 +5283,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_adds_id_key_when_configured
     driver.configure("id_key request_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4842,6 +5294,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_id_key_with_dot
       driver.configure("id_key nested.request_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4851,6 +5304,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_id_key_with_dollar_dot
       driver.configure("id_key $.nested.request_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4860,6 +5314,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_id_key_with_bracket
       driver.configure("id_key $['nested']['request_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4870,6 +5325,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4878,6 +5334,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_adds_id_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4887,6 +5344,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_adds_parent_key_when_configured
     driver.configure("parent_key parent_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4897,6 +5355,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_dot
       driver.configure("parent_key nested.parent_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4906,6 +5365,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_dollar_dot
       driver.configure("parent_key $.nested.parent_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4915,6 +5375,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_parent_key_with_bracket
       driver.configure("parent_key $['nested']['parent_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4925,6 +5386,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_doesnt_add_parent_key_if_missing_when_configured
     driver.configure("parent_key another_parent_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4933,6 +5395,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_adds_parent_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4943,6 +5406,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_es6
       driver("routing_key routing_id\n", 6)
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
@@ -4952,6 +5416,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_es7
       driver("routing_key routing_id\n", 7)
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
@@ -4963,6 +5428,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_routing_key_with_dot
       driver.configure("routing_key nested.routing_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4972,6 +5438,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_routing_key_with_dollar_dot
       driver.configure("routing_key $.nested.routing_id\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4981,6 +5448,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     def test_adds_nested_routing_key_with_bracket
       driver.configure("routing_key $['nested']['routing_id']\n")
       stub_elastic
+      stub_elastic_info
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
@@ -4991,6 +5459,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_doesnt_add_routing_key_if_missing_when_configured
     driver.configure("routing_key another_routing_id\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -4999,6 +5468,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_adds_routing_key_when_not_configured
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5008,6 +5478,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_remove_one_key
     driver.configure("remove_keys key1\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
     end
@@ -5018,6 +5489,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_remove_multi_keys
     driver.configure("remove_keys key1, key2\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
     end
@@ -5026,6 +5498,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   end
 
   def test_request_error
+    stub_elastic_info
     stub_elastic_unavailable
     assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
@@ -5037,6 +5510,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_request_forever
     omit("retry_forever test is unstable.") if ENV["CI"]
     stub_elastic
+    stub_elastic_info
     driver.configure(Fluent::Config::Element.new(
                'ROOT', '', {
                  '@type' => 'elasticsearch',
@@ -5061,6 +5535,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
+    stub_elastic_info
 
     assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
@@ -5077,6 +5552,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
+    stub_elastic_info
 
     driver.configure("reconnect_on_error true\n")
 
@@ -5103,6 +5579,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
+    stub_elastic_info
 
     driver.configure("reconnect_on_error false\n")
 
@@ -5146,6 +5623,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         })
      }
     end)
+    stub_elastic_info
 
     driver.run(default_tag: 'test') do
       driver.feed(1, sample_record)
@@ -5217,6 +5695,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         })
                      }
                    end)
+      stub_elastic_info
 
       # Check buffer fulfillment condition
       assert_raise(Fluent::Plugin::ElasticsearchOutput::RetryStreamEmitFailure) do
@@ -5263,6 +5742,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         })
      }
     end)
+    stub_elastic_info
+
     sample_record1 = sample_record('my_id' => 'abc')
     sample_record4 = sample_record('my_id' => 'xyz')
 
@@ -5312,6 +5793,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         })
      }
     end)
+    stub_elastic_info
+
     sample_record1 = sample_record('my_id' => 'abc')
     sample_record4 = sample_record('my_id' => 'xyz')
 
@@ -5382,6 +5865,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         })
      }
     end)
+    stub_elastic_info
 
     driver.run(default_tag: 'test') do
       driver.feed(1, sample_record)
@@ -5398,6 +5882,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5407,6 +5892,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_upsert_should_not_write_if_theres_no_id
     driver.configure("write_operation upsert\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5416,6 +5902,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_create_should_not_write_if_theres_no_id
     driver.configure("write_operation create\n")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5426,6 +5913,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("write_operation update
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5439,6 +5927,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       id_key request_id
                       remove_keys_on_update parent_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5450,6 +5939,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5463,6 +5953,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       id_key request_id
                       remove_keys_on_update parent_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5477,6 +5968,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       id_key request_id
                       remove_keys_on_update parent_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5490,6 +5982,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       id_key id
                       remove_keys_on_update foo,baz")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
     end
@@ -5514,6 +6007,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       id_key id
                       remove_keys_on_update_key keys_to_skip")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
     end
@@ -5538,6 +6032,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       remove_keys_on_update foo,bar
                       remove_keys_on_update_key keys_to_skip")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
     end
@@ -5562,6 +6057,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure("write_operation create
                       id_key request_id")
     stub_elastic
+    stub_elastic_info
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
@@ -5570,6 +6066,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_include_index_in_url
     stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
+    stub_elastic_info('http://localhost:9200/')
 
     driver.configure("index_name logstash-2018.01.01
                       include_index_in_url true")
@@ -5583,8 +6080,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
   def test_use_simple_sniffer
     require 'fluent/plugin/elasticsearch_simple_sniffer'
-    stub_elastic_info
     stub_elastic
+    stub_elastic_info
     config = %[
       sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
       log_level debug
@@ -5608,6 +6105,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       remove_keys id
                       suppress_doc_wrap true')
     stub_elastic
+    stub_elastic_info
     doc_body = {'field' => 'value'}
     script_body = {'source' => 'ctx._source.counter += params.param1',
                    'lang' => 'painless',
@@ -5634,6 +6132,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
                       remove_keys id
                       suppress_doc_wrap true')
     stub_elastic
+    stub_elastic_info
     doc_body = {'field' => 'value'}
     script_body = {'source' => 'ctx._source.counter += params.param1',
                    'lang' => 'painless',
@@ -5660,6 +6159,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_ignore_exception
     driver.configure('ignore_exceptions ["Elasticsearch::Transport::Transport::Errors::ServiceUnavailable"]')
     stub_elastic_unavailable
+    stub_elastic_info
 
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -5669,6 +6169,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_ignore_exception_with_superclass
     driver.configure('ignore_exceptions ["Elasticsearch::Transport::Transport::ServerError"]')
     stub_elastic_unavailable
+    stub_elastic_info
 
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -5678,6 +6179,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_ignore_excetion_handles_appropriate_ones
     driver.configure('ignore_exceptions ["Faraday::ConnectionFailed"]')
     stub_elastic_unavailable
+    stub_elastic_info
 
     assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do