fluent-plugin-elasticsearch 5.1.5 → 5.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ae6214f54783c402097e26276b9247f1d58866b08cdc80be47d3b4638516d54
-  data.tar.gz: b69af374d04f84b2b1137189916c2b78f25eae71c2294d9a0cc196c30c09db29
+  metadata.gz: 62c61f26ed869460900c3f05d6c70bc4bbb06dd9747f3a23c92accd4c878756f
+  data.tar.gz: 1ed0894c48a957b5359739303b8fef7e9df9ce8ec80b1b0101b2b17d40a72967
 SHA512:
-  metadata.gz: 29173c2331bc0e533aa1d3aa66aa555bad7802e5fbb4207641b11aebc0a18c0d4e44ebed0b167c8fe4bddf403d2dd5a61349afeffecfc28a65359c3bb4cce26b
-  data.tar.gz: 7fdf58624182bdd4e354cad2d4f9ba001cff5a8c8c0ba02ac043607a87e2372311deb85bc0306d3aa167e98170a6049f33f3203b202ebcf25eac75e65dc23a17
+  metadata.gz: c63a8ea452e6f3691497e4601b46393c851ae188b9583ebfeaacff507e7791206430145db01bea4298d7c0021d610e9e2a229da9cca5394bea731ae36e204b9b
+  data.tar.gz: 1e4ed30e6d00e372781d8744262bc711937ddaffdfc92f9f0407c7f367f0e4ac7807ba635a119b3287a09d2780c9d7efbe2e1e031330a0aacd527a4ff2a177bd

data/Gemfile

@@ -7,5 +7,5 @@ gem 'simplecov', require: false
 gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
-gem "elasticsearch-xpack"
+gem "elasticsearch-xpack" if ENV["USE_XPACK"]
 gem "oj"

data/History.md

@@ -2,6 +2,18 @@
 
 ### [Unreleased]
 
+### 5.2.2
+-  Add missing top level class markers (#961)
+- Ensure use_record_as_seed for same records (#960)
+
+### 5.2.1
+- respect include\_tag\_key and tag\_key setting when using data streams (#936)
+- Handle unsupported version error (#956)
+- Display deprecated warning on ES dynamic plugin (#955)
+
+### 5.2.0
+-  Migrate to handle Elasticsearch 8 (#949)
+
 ### 5.1.5
 - Make retryable DataStreams creation at configure phase (#943)
 - Handle @hosts parameter on data_stream plugin (#942)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.1.5'
+  s.version       = '5.2.2'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_compat.rb

@@ -0,0 +1,30 @@
+begin
+  require 'elastic/transport'
+  ::TRANSPORT_CLASS = Elastic::Transport
+rescue LoadError
+end
+begin
+  require 'elasticsearch/transport'
+  ::TRANSPORT_CLASS = Elasticsearch::Transport
+rescue LoadError
+end
+if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
+  begin
+    require 'elasticsearch/xpack'
+  rescue LoadError
+  end
+end
+
+begin
+  require 'elastic/transport/transport/connections/selector'
+  ::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
+rescue LoadError
+end
+begin
+  require 'elasticsearch/transport/transport/connections/selector'
+  ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
+rescue LoadError
+end
+unless defined?(::Elasticsearch::UnsupportedProductError)
+  class ::Elasticsearch::UnsupportedProductError < StandardError; end
+end

data/lib/fluent/plugin/elasticsearch_fallback_selector.rb

@@ -1,7 +1,7 @@
-require 'elasticsearch/transport/transport/connections/selector'
+require_relative 'elasticsearch_compat'
 
 class Fluent::Plugin::ElasticseatchFallbackSelector
-  include Elasticsearch::Transport::Transport::Connections::Selector::Base
+  include SELECTOR_CLASS::Base
 
   def select(options={})
     connections.first

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -1,3 +1,5 @@
+require_relative 'elasticsearch_compat'
+
 module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
 
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
 
-    rescue Elasticsearch::Transport::Transport::Error => e
+    rescue ::TRANSPORT_CLASS::Transport::Error => e
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
     end
   end
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   end
 
   def get_ilm_policy
-    client.ilm.get_policy
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+      client.ilm.get_policy
+    else
+      client.enrich.get_policy
+    end
   end
 
   def ilm_policy_exists?(policy_id)
     begin
-      client.ilm.get_policy(policy_id: policy_id)
+      if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+        client.ilm.get_policy(policy_id: policy_id)
+      else
+        client.enrich.get_policy(name: policy_id)
+      end
       true
     rescue
       false
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def ilm_policy_put(policy_id, policy)
     log.info("Installing ILM policy: #{policy}")
-    client.ilm.put_policy(policy_id: policy_id, body: policy)
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+      client.ilm.put_policy(policy_id: policy_id, body: policy)
+    else
+      client.enrich.put_policy(name: policy_id, body: policy)
+    end
   end
 
   def default_policy_payload

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -28,12 +28,16 @@ module Fluent::ElasticsearchIndexTemplate
       client(host).indices.get_index_template(:name => name)
     end
     return true
-  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+  rescue TRANSPORT_CLASS::Transport::Errors::NotFound
     return false
   end
 
   def host_unreachable_exceptions
-    if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+    if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      # elasticsearch-ruby 8.0.0's elastic-transport uses
+      # direct callable #host_unreachable_exceptions again.
+      client.transport.host_unreachable_exceptions
+    elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
       # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
       # Elasticsearch class on Transport.
       # This is why #host_unreachable_exceptions is not callable directly
@@ -47,7 +51,7 @@ module Fluent::ElasticsearchIndexTemplate
   def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
-    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
+    transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
     rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
@@ -78,7 +82,7 @@ module Fluent::ElasticsearchIndexTemplate
 
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
-  rescue Elasticsearch::Transport::Transport::Error => e
+  rescue TRANSPORT_CLASS::Transport::Error => e
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -1,6 +1,7 @@
 require 'elasticsearch'
+require_relative 'elasticsearch_compat'
 
-class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
+class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
 
   def hosts
     @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -53,7 +53,7 @@ module Fluent::Plugin
       seed += tag + separator if @include_tag_in_seed
       seed += time.to_s + separator if @include_time_in_seed
       if @use_entire_record
-        record.each {|k,v| seed += "|#{k}|#{v}"}
+        record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
       else
         seed += record_keys.map {|k| record[k]}.join(separator)
       end

data/lib/fluent/plugin/in_elasticsearch.rb

@@ -3,6 +3,7 @@ require 'elasticsearch'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
 require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   class ElasticsearchInput < Input
@@ -218,7 +219,7 @@ module Fluent::Plugin
 
         headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
 
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
           connection_options.merge(
             options: {
               reload_connections: local_reload_connections,

data/lib/fluent/plugin/oj_serializer.rb

@@ -1,10 +1,11 @@
 require 'oj'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   module Serializer
 
     class Oj
-      include Elasticsearch::Transport::Transport::Serializer::Base
+      include TRANSPORT_CLASS::Transport::Serializer::Base
 
       # De-serialize a Hash from JSON string
       #

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -3,10 +3,6 @@ require 'date'
 require 'excon'
 require 'elasticsearch'
 require 'set'
-begin
-  require 'elasticsearch/xpack'
-rescue LoadError
-end
 require 'json'
 require 'uri'
 require 'base64'
@@ -23,6 +19,7 @@ require 'fluent/time'
 require 'fluent/unique_id'
 require 'fluent/log-ext'
 require 'zlib'
+require_relative 'elasticsearch_compat'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
@@ -413,11 +410,11 @@ EOC
         end
       end
 
-      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+      if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
         if compression
           raise Fluent::ConfigError, <<-EOC
             Cannot use compression with elasticsearch-transport plugin version < 7.2.0
-            Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
+            Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
             Please consider to upgrade ES client.
           EOC
         end
@@ -495,7 +492,11 @@ EOC
     end
 
     def detect_es_major_version
-      @_es_info ||= client.info
+      begin
+        @_es_info ||= client.info
+      rescue ::Elasticsearch::UnsupportedProductError => e
+        raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+      end
       begin
         unless version = @_es_info.dig("version", "number")
           version = @default_elasticsearch_version
@@ -613,7 +614,7 @@ EOC
                     .merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: local_reload_connections,
                                                                               reload_on_failure: @reload_on_failure,

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -23,11 +23,19 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      begin
-        require 'elasticsearch/api'
-        require 'elasticsearch/xpack'
-      rescue LoadError
-        raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
+      if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+        begin
+          require 'elasticsearch/api'
+          require 'elasticsearch/xpack'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
+        end
+      else
+        begin
+          require 'elasticsearch/api'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
+        end
       end
 
       @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
@@ -89,13 +97,16 @@ module Fluent::Plugin
       end
 
       params = {
-        policy_id: ilm_name,
         body: @data_stream_ilm_policy
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        client(host).xpack.ilm.put_policy(params)
+        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).enrich.put_policy(params.merge(name: ilm_name))
+        else
+          client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
+        end
       end
     end
 
@@ -127,8 +138,8 @@ module Fluent::Plugin
       }
       begin
         response = client(host).indices.get_data_stream(params)
-        return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+        return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
+      rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
         log.info "Specified data stream does not exist. Will be created: <#{e}>"
         return false
       end
@@ -148,7 +159,11 @@ module Fluent::Plugin
 
     def ilm_policy_exists?(policy_id, host = nil)
       begin
-        client(host).ilm.get_policy(policy_id: policy_id)
+        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).enrich.get_policy(name: policy_id)
+        else
+          client(host).ilm.get_policy(policy_id: policy_id)
+        end
         true
       rescue
         false
@@ -162,7 +177,7 @@ module Fluent::Plugin
         client(host).indices.get_index_template(:name => name)
       end
       return true
-    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    rescue TRANSPORT_CLASS::Transport::Errors::NotFound
       return false
     end
 
@@ -228,6 +243,10 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+
         begin
           record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -28,6 +28,8 @@ module Fluent::Plugin
         @dynamic_config[key] = value.to_s
       }
       # end eval all configs
+
+      log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
     end
 
     def create_meta_config_map
@@ -53,7 +55,7 @@ module Fluent::Plugin
                        end
         headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: @reload_connections,
                                                                               reload_on_failure: @reload_on_failure,

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -18,17 +18,25 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
-  def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [404, "Not Found"])
+  def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
-  def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
+  def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
   def sample_record(content={})
@@ -60,7 +68,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
       reload_after 10
       catch_transport_exception_on_retry false # For fallback testing
     ]
-    assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
+    assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
       driver(config)
     end
     driver.run(default_tag: 'test') do

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -6,12 +6,14 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def setup
-    begin
-      require "elasticsearch/xpack"
-    rescue LoadError
-      omit "ILM testcase needs elasticsearch-xpack gem."
+    if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("7.14.0")
+      begin
+        require "elasticsearch/xpack"
+      rescue LoadError
+        omit "ILM testcase needs elasticsearch-xpack gem."
+      end
     end
-    if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+    if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.4.0")
       omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
     end
     Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
@@ -27,50 +29,78 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def ilm_existence_endpoint(policy_id)
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      "_enrich/policy/#{policy_id}"
+    else
+      "_ilm/policy/%7B:policy_id=%3E%22#{policy_id}%22%7D"
+    end
+  end
+
+  def ilm_creation_endpoint(policy_id)
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      "_enrich/policy/#{policy_id}"
+    else
+      "_ilm/policy/#{policy_id}"
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
     stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
-    assert_equal(expected, xpack_info)
+    if xpack_info.is_a?(Elasticsearch::API::Response)
+      assert_equal(expected, xpack_info.body)
+    else
+      assert_equal(expected, xpack_info)
+    end
   end
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
     stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
       to_return(:status => 404, :body => "", :headers => {})
     stub_elastic_info
-    assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
+    assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
       to_return(:status => 200, :body => "", :headers => {})
     stub_elastic_info
-    assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
+    assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
-    stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
+    stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
+    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
       to_return(:status => 200, :body => "", :headers => {})
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}", times: 1)
   end
 end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -132,10 +132,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
   end
 
   class UseEntireRecordAsSeedTest < self
-    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
-         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
-         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
-         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    data("md5" => ["md5", "OAod7J0DR9s9/rOQnkeSFw=="],
+         "sha1" => ["sha1", "0CT4aMJ4gxMT3TKaYPCYApiVsq8="],
+         "sha256" => ["sha256", "mbAuKF5can0TTj/JBk71AXtOyoVqw5W5gMPUxx6pxLk="],
+         "sha512" => ["sha512", "f7kz5KVuDy+riENePDzqBjGQfbuRNpRBSQMzT2/6hrljXbYtBy3YFmxB86ofIf3zz4ZBao2QM2W7YvcwbRtK1w=="],)
     def test_record
       hash_type, expected = data
       d = create_driver(%[
@@ -151,10 +151,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
-         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
-         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
-         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    data("md5" => ["md5", "Hb0jwxofNQP+ufQTKK1U4g=="],
+         "sha1" => ["sha1", "BakTtlotl/u+yOON6YcViTz6nms="],
+         "sha256" => ["sha256", "eLuTCsFqDlk6PfABNyD39r36+yNIBeDTHyNKfJ8fZQw="],
+         "sha512" => ["sha512", "PhPCNGalM4H4xT19DnCBnpwr56lbvCo8wJGyCiH9dWcyhn1nA5l1diYSZlF2fNiq1+wzMqfGvJILIjgQrlAPcg=="],)
     def test_record_with_tag
       hash_type, expected = data
       d = create_driver(%[
@@ -171,10 +171,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
-         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
-         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
-         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    data("md5" => ["md5", "C8vfhC4kecNCNutFCuC6MA=="],
+         "sha1" => ["sha1", "+YWVqUEL90wpKJRrionUJwNgXHg="],
+         "sha256" => ["sha256", "eSqGZqjnO6Uum/4CNfJaolX49+2XKogiGMHGNHiO91Q="],
+         "sha512" => ["sha512", "iVmuD0D+i/WtBwNza09ZXSIW8Xg8/yrUwK/M/EZaCMjz/x5FyyCiVkb1VVKsgNnJy0SYt4w21dhHewu1aXM6HA=="],)
     def test_record_with_time
       hash_type, expected = data
       d = create_driver(%[
@@ -191,10 +191,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
-         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
-         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
-         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    data("md5" => ["md5", "lU7d4EiF+2M1zxWcsmBbjg=="],
+         "sha1" => ["sha1", "nghmz1y3KTEFxalfS2/Oe4n4yfQ="],
+         "sha256" => ["sha256", "d0le9UOnUeuGPF/2yEBRM1YzOYeHtxYOE1UU6JgJrvU="],
+         "sha512" => ["sha512", "n7rhisGHUBne6c4Cs9DRMbPror8O5Y/vYajDqAtOaiUTys/Z1EKBMnZQA0iVNFw7joX33cenBW3Yyccct3xSew=="],)
     def test_record_with_tag_and_time
       hash_type, expected = data
       d = create_driver(%[

data/test/plugin/test_in_elasticsearch.rb

@@ -31,9 +31,17 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '7.9.0'.freeze
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def sample_response(index_name="fluentd")