fluent-plugin-elasticsearch 5.0.0 → 5.2.3

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -28,6 +28,8 @@ module Fluent::Plugin
         @dynamic_config[key] = value.to_s
       }
       # end eval all configs
+
+      log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
     end
 
     def create_meta_config_map
@@ -53,7 +55,7 @@ module Fluent::Plugin
                        end
         headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: @reload_connections,
                                                                               reload_on_failure: @reload_on_failure,

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'
+require 'msgpack'
 
 class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
@@ -27,13 +28,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
+        raise 'process_message'
       end
       return true
     end
@@ -49,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class MockMsgpackChunk
+    def initialize(chunk)
+      @chunk = chunk
+      @factory = MessagePack::Factory.new
+      @factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+    end
+
+    def msgpack_each
+      @factory.unpacker(@chunk).each { |time, record| yield(time, record) }
+    end
+  end
+
+  class MockUnpackedMsg
+    def initialize(records)
+      @records = records
+    end
+    def each
+      @records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
+    end
+  end
+
   def setup
     Fluent::Test.setup
     @log_device = Fluent::Test::DummyLogDevice.new
@@ -93,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -135,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -154,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       "items" : [{}]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(0, @plugin.error_events.size)
     assert_nil(@plugin.error_events[0])
   end
@@ -176,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -199,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -225,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -252,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -285,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -302,11 +337,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   def test_retry_error
     records = []
     error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
+    error_records.merge!({0=>true, 4=>true})
     10.times do |i|
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -386,6 +422,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
               "reason":"unrecognized error"
             }
           }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "9",
+            "status" : 500,
+            "error" : {
+              "type" : "json_parse_exception",
+              "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
+            }
+          }
         }
       ]
     }))
@@ -393,19 +441,19 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
         next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 2, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 8, records[1]['_id']
+      assert_equal 2, records.length, "Exp. retry_stream to contain records"
+      assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
+      assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 3, error_ids.length
-      assert_equal [5, 6, 7], error_ids
+      assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
+      assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end
@@ -422,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -509,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -532,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -622,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -643,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
     assert_true failed
   end
+
+  def test_nested_msgpack_each
+    cwd = File.dirname(__FILE__)
+    chunk_path = File.join(cwd, 'mock_chunk.dat')
+    chunk_file = File.open(chunk_path, 'rb', 0644)
+    chunk_file.seek(0, IO::SEEK_SET)
+
+    chunk = MockMsgpackChunk.new(chunk_file)
+
+    unpacked_msg_arr = []
+    msg_count = 0
+    count_to_trigger_error_handle = 0
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
+
+      unpacked_msg_arr << {:time => time, :record => record}
+      msg_count += 1
+
+      record.each_key do |k|
+        if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
+          assert_equal(:impossible, k)
+        end
+      end
+
+      if msg_count % 55 == 0
+        if count_to_trigger_error_handle == 1
+          begin
+            response = {}
+            response['errors'] = true
+            response['items'] = []
+            item = {}
+            item['index'] = {}
+            item['index']['status'] = 429
+            item['index']['error'] = {}
+            item['index']['error']['type'] = "es_rejected_execution_exception"
+            abc = 0
+            while abc < unpacked_msg_arr.length
+              abc += 1
+              response['items'] << item
+            end
+
+            dummy_extracted_values = []
+            @handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
+            assert_equal(0, @plugin.error_events.size)
+            assert_nil(@plugin.error_events[0])
+          rescue => e
+            # capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
+          end
+        end
+
+        count_to_trigger_error_handle += 1
+        unpacked_msg_arr.clear
+      end # end if
+    end # end chunk.msgpack_each
+
+    chunk_file.close
+  end
 end

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -18,17 +18,25 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
-  def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [404, "Not Found"])
+  def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
-  def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
+  def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
   def sample_record(content={})
@@ -60,7 +68,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
       reload_after 10
       catch_transport_exception_on_retry false # For fallback testing
     ]
-    assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
+    assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
       driver(config)
     end
     driver.run(default_tag: 'test') do

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -6,12 +6,14 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def setup
-    begin
-      require "elasticsearch/xpack"
-    rescue LoadError
-      omit "ILM testcase needs elasticsearch-xpack gem."
+    if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("7.14.0")
+      begin
+        require "elasticsearch/xpack"
+      rescue LoadError
+        omit "ILM testcase needs elasticsearch-xpack gem."
+      end
     end
-    if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+    if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.4.0")
       omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
     end
     Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
@@ -27,40 +29,78 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def ilm_existence_endpoint(policy_id)
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      "_enrich/policy/#{policy_id}"
+    else
+      "_ilm/policy/%7B:policy_id=%3E%22#{policy_id}%22%7D"
+    end
+  end
+
+  def ilm_creation_endpoint(policy_id)
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      "_enrich/policy/#{policy_id}"
+    else
+      "_ilm/policy/#{policy_id}"
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+    stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
-    assert_equal(expected, xpack_info)
+    if xpack_info.is_a?(Elasticsearch::API::Response)
+      assert_equal(expected, xpack_info.body)
+    else
+      assert_equal(expected, xpack_info)
+    end
   end
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+    stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
       to_return(:status => 404, :body => "", :headers => {})
-    assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
+    stub_elastic_info
+    assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
       to_return(:status => 200, :body => "", :headers => {})
-    assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
+    stub_elastic_info
+    assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
-    stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
+    stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
+    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     create_ilm_policy("fluent-policy")
 
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}", times: 1)
   end
 end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -132,10 +132,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
   end
 
   class UseEntireRecordAsSeedTest < self
-    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
-         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
-         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
-         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    data("md5" => ["md5", "OAod7J0DR9s9/rOQnkeSFw=="],
+         "sha1" => ["sha1", "0CT4aMJ4gxMT3TKaYPCYApiVsq8="],
+         "sha256" => ["sha256", "mbAuKF5can0TTj/JBk71AXtOyoVqw5W5gMPUxx6pxLk="],
+         "sha512" => ["sha512", "f7kz5KVuDy+riENePDzqBjGQfbuRNpRBSQMzT2/6hrljXbYtBy3YFmxB86ofIf3zz4ZBao2QM2W7YvcwbRtK1w=="],)
     def test_record
       hash_type, expected = data
       d = create_driver(%[
@@ -151,10 +151,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
-         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
-         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
-         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    data("md5" => ["md5", "Hb0jwxofNQP+ufQTKK1U4g=="],
+         "sha1" => ["sha1", "BakTtlotl/u+yOON6YcViTz6nms="],
+         "sha256" => ["sha256", "eLuTCsFqDlk6PfABNyD39r36+yNIBeDTHyNKfJ8fZQw="],
+         "sha512" => ["sha512", "PhPCNGalM4H4xT19DnCBnpwr56lbvCo8wJGyCiH9dWcyhn1nA5l1diYSZlF2fNiq1+wzMqfGvJILIjgQrlAPcg=="],)
     def test_record_with_tag
       hash_type, expected = data
       d = create_driver(%[
@@ -171,10 +171,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
-         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
-         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
-         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    data("md5" => ["md5", "C8vfhC4kecNCNutFCuC6MA=="],
+         "sha1" => ["sha1", "+YWVqUEL90wpKJRrionUJwNgXHg="],
+         "sha256" => ["sha256", "eSqGZqjnO6Uum/4CNfJaolX49+2XKogiGMHGNHiO91Q="],
+         "sha512" => ["sha512", "iVmuD0D+i/WtBwNza09ZXSIW8Xg8/yrUwK/M/EZaCMjz/x5FyyCiVkb1VVKsgNnJy0SYt4w21dhHewu1aXM6HA=="],)
     def test_record_with_time
       hash_type, expected = data
       d = create_driver(%[
@@ -191,10 +191,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
-         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
-         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
-         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    data("md5" => ["md5", "lU7d4EiF+2M1zxWcsmBbjg=="],
+         "sha1" => ["sha1", "nghmz1y3KTEFxalfS2/Oe4n4yfQ="],
+         "sha256" => ["sha256", "d0le9UOnUeuGPF/2yEBRM1YzOYeHtxYOE1UU6JgJrvU="],
+         "sha512" => ["sha512", "n7rhisGHUBne6c4Cs9DRMbPror8O5Y/vYajDqAtOaiUTys/Z1EKBMnZQA0iVNFw7joX33cenBW3Yyccct3xSew=="],)
     def test_record_with_tag_and_time
       hash_type, expected = data
       d = create_driver(%[

data/test/plugin/test_in_elasticsearch.rb

@@ -31,6 +31,19 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '7.9.0'.freeze
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
   def sample_response(index_name="fluentd")
     {
       "took"=>4,
@@ -322,6 +335,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG)
     driver.run(expect_emits: 1, timeout: 10)
@@ -337,6 +351,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
     driver.run(expect_emits: 1, timeout: 10)
@@ -352,6 +367,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
     driver.run(expect_emits: 1, timeout: 10)
@@ -370,6 +386,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
                       timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
@@ -389,6 +406,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
     driver.run(expect_emits: 1, timeout: 10)
@@ -412,6 +430,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
     driver.run(expect_emits: 1, timeout: 10)
@@ -434,6 +453,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1
     end
+    stub_elastic_info
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,