fluent-plugin-elasticsearch 4.2.1 → 4.3.3

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -58,6 +58,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
       with_transporter_log true
       reload_connections true
       reload_after 10
+      catch_transport_exception_on_retry false # For fallback testing
     ]
     assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
       driver(config)

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -11,6 +11,9 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     rescue LoadError
       omit "ILM testcase needs elasticsearch-xpack gem."
     end
+    if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+      omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+    end
     Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
       def client
         Elasticsearch::Client.new url: 'localhost:9200'

data/test/plugin/test_out_elasticsearch.rb

@@ -322,6 +322,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal true, instance.client(nil, compressable).transport.options[:compression]
   end
 
+  test 'check configure cloud_id based client' do
+
+    config = %{
+      cloud_id "name:bG9jYWxob3N0JGFiY2QkZWZnaA=="
+      cloud_auth "some:auth"
+    }
+    instance = driver(config).instance
+
+    assert_equal "name:bG9jYWxob3N0JGFiY2QkZWZnaA==", instance.cloud_id
+    assert_equal "some", instance.user
+    assert_equal "auth", instance.password
+  end
+
   test 'configure Content-Type' do
     config = %{
       content_type application/x-ndjson
@@ -357,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     }
   end
 
+  sub_test_case 'Check client.info response' do
+    def create_driver(conf='', es_version=5, client_version="\"5.0\"")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      @default_elasticsearch_version ||= es_version
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          @_es_info ||= client.info
+          begin
+            unless version = @_es_info.dig("version", "number")
+              version = @default_elasticsearch_version
+            end
+          rescue NoMethodError => e
+            log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
+            version = @default_elasticsearch_version
+          end
+          version.to_i
+        end
+      CODE
+
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
+        # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+        if !defined?(Fluent::Plugin::Output)
+          def format(tag, time, record)
+            [time, record].to_msgpack
+          end
+        end
+      }.configure(conf)
+    end
+
+    def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
+      body ="{\"version\":{\"number\":\"#{version}\"}}"
+      stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
+    end
+
+    test 'handle invalid client.info' do
+      stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
+      config = %{
+        host     logs.fluentd.com
+        port     24225
+        scheme   https
+        path     /es/
+        user     john
+        password doe
+        default_elasticsearch_version 7
+        scheme https
+        @log_level info
+      }
+      d = create_driver(config, 7, "\"7.10.1\"")
+      logs = d.logs
+      assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
+    end
+  end
+
   sub_test_case 'Check TLS handshake stuck warning log' do
     test 'warning TLS log' do
       config = %{
@@ -383,7 +455,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     test 'valid configuration of index lifecycle management' do |data|
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
 
@@ -419,7 +497,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     test 'valid configuration of overwriting ilm_policy' do |data|
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
 
@@ -823,6 +907,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_already_present(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     config = %{
       host            logs.google.com
       port            777
@@ -853,6 +940,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -894,6 +984,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_with_rollover_index_and_template_related_placeholders(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -959,6 +1052,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -1034,12 +1130,18 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       rescue LoadError
         omit "ILM testcase needs elasticsearch-xpack gem."
       end
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
     end
 
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1120,6 +1222,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1209,6 +1314,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1316,6 +1424,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1396,6 +1507,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_custom_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1477,6 +1591,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_ilm_policies_and_placeholderstest_template_create_with_rollover_index_and_ilm_policies_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1565,6 +1682,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
            "new_template"    => [false, "_index_template"])
       def test_tag_placeholder(data)
         use_legacy_template_flag, endpoint = data
+        if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+          omit "elastisearch-ruby v7.8.0 or later is needed."
+        end
         cwd = File.dirname(__FILE__)
         template_file = if use_legacy_template_flag
                           File.join(cwd, 'test_template.json')
@@ -1652,6 +1772,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
            "new_template"    => [false, "_index_template"])
       def test_tag_placeholder_with_multiple_policies(data)
         use_legacy_template_flag, endpoint = data
+        if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+          omit "elastisearch-ruby v7.8.0 or later is needed."
+        end
         cwd = File.dirname(__FILE__)
         template_file = if use_legacy_template_flag
                           File.join(cwd, 'test_template.json')
@@ -1740,6 +1863,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1775,30 +1901,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
       if use_legacy_template_flag
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag1-*\",\"order\":52}").
           to_return(status: 200, body: "", headers: {})
       else
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"index_patterns\":\"logstash-test-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+               body: "{\"index_patterns\":\"logstash-tag1-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
           to_return(status: 200, body: "", headers: {})
       end
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag1").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash-tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
         with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-test").
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag2-*\",\"order\":52}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"index_patterns\":\"logstash-tag2-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+          to_return(status: 200, body: "", headers: {})
+      end      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag2").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"logstash-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"logstash-tag2\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -1815,11 +1976,107 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
 
       elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
       driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
+        driver.feed('tag1', event_time, sample_record)
+        driver.feed('tag2', event_time, sample_record)
+        driver.feed('tag1', event_time, sample_record)
+        driver.feed('tag2', event_time, sample_record)
       end
-      assert_equal('logstash-test', index_cmds.first['index']['_index'])
+      assert_equal('logstash-tag2', index_cmds.first['index']['_index'])
 
-      assert_equal ["logstash-test"], driver.instance.alias_indexes
+      assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
+
+      assert_requested(elastic_request)
+    end
+
+
+    data("legacy_template" => [true, "_template"],
+         "new_template"    => [false, "_index_template"])
+    def test_template_create_with_rollover_index_and_default_ilm_and_placeholders_and_index_separator(data)
+      use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
+      cwd = File.dirname(__FILE__)
+      template_file = if use_legacy_template_flag
+                        File.join(cwd, 'test_template.json')
+                      else
+                        File.join(cwd, 'test_index_template.json')
+                      end
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        index_date_pattern now/w{xxxx.ww}
+        index_name logstash.${tag}
+        enable_ilm      true
+        index_separator "."
+        use_legacy_template #{use_legacy_template_flag}
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash.tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash.tag1.*\",\"order\":52}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"index_patterns\":\"logstash.tag1.*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
+          to_return(status: 200, body: "", headers: {})
+      end
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash.tag1").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash.tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed('tag1', event_time, sample_record)
+      end
+      assert_equal('logstash.tag1', index_cmds.first['index']['_index'])
+
+      assert_equal ["logstash.tag1"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -1828,6 +2085,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1922,6 +2182,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_create(data)
     use_legacy_template_flag, endpoint = data
+
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -1964,6 +2228,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_create_with_customize_template_related_placeholders(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2014,6 +2281,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_installation_for_host_placeholder(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2060,6 +2330,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2118,6 +2391,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create_and_deflector_alias(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2177,6 +2453,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create_with_logstash_format(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2202,7 +2481,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       use_legacy_template #{use_legacy_template_flag}
     }
 
-    timestr = Time.now.strftime("%Y.%m.%d")
+    timestr = Time.now.getutc.strftime("%Y.%m.%d")
     # connection start
     stub_request(:head, "https://logs.google.com:777/es//").
       with(basic_auth: ['john', 'doe']).
@@ -2245,12 +2524,18 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       rescue LoadError
         omit "ILM testcase needs elasticsearch-xpack gem."
       end
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed."
+      end
     end
 
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2335,6 +2620,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2449,6 +2737,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2478,6 +2769,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       stub_request(:head, "https://logs.google.com:777/es//").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
+
+      # test-tag1
       # check if template exists
       stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
         with(basic_auth: ['john', 'doe']).
@@ -2487,31 +2780,78 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # creation of index which can rollover
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
       if use_legacy_template_flag
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
+               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag1-*\"}").
           to_return(status: 200, body: "", headers: {})
       else
-        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
           with(basic_auth: ['john', 'doe'],
-               body: "{\"priority\":108,\"index_patterns\":\"mylogs-custom-test-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
+               body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag1-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
           to_return(status: 200, body: "", headers: {})
       end
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag1").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"mylogs-custom-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"mylogs-test-tag1\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      # test-tag2
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # creation of index which can rollover
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      if use_legacy_template_flag
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag2-*\"}").
+          to_return(status: 200, body: "", headers: {})
+      else
+        stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag2-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
+          to_return(status: 200, body: "", headers: {})
+      end
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag2").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"mylogs-test-tag2\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -2527,12 +2867,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       driver(config)
 
       elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'custom-test') do
-        driver.feed(sample_record)
+      driver.run(default_tag: 'test') do
+        driver.feed("test-tag1", event_time, sample_record)
+        driver.feed("test-tag2", event_time, sample_record)
+        driver.feed("test-tag1", event_time, sample_record)
+        driver.feed("test-tag2", event_time, sample_record)
       end
-      assert_equal('mylogs-custom-test', index_cmds.first['index']['_index'])
-
-      assert_equal ["mylogs-custom-test"], driver.instance.alias_indexes
+      assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
+      assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -2541,6 +2883,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_custom_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2619,6 +2964,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2661,6 +3009,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2704,6 +3055,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2790,6 +3144,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_for_host_placeholder(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2838,6 +3195,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_retry_install_fails(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2873,10 +3233,54 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(4, connection_resets)
   end
 
+  transport_errors_handled_separately = [Elasticsearch::Transport::Transport::Errors::NotFound]
+  transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map { |err| [err, Elasticsearch::Transport::Transport::Errors.const_get(err)]  }
+  transport_errors_hash = Hash[transport_errors.select { |err| !transport_errors_handled_separately.include?(err[1]) } ]
+
+  data(transport_errors_hash)
+  def test_template_retry_transport_errors(error)
+    endpoint, use_legacy_template_flag = if Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.8.0")
+                 ["_index_template".freeze, false]
+               else
+                 ["_template".freeze, true]
+               end
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_index_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            778
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      max_retry_putting_template 0
+      use_legacy_template #{use_legacy_template_flag}
+    }
+
+    retries = 0
+    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
+      .with(basic_auth: ['john', 'doe']) do |req|
+      retries += 1
+      raise error
+    end
+
+    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
+      driver(config)
+    end
+
+    assert_equal(1, retries)
+  end
+
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_template_retry_install_does_not_fail(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2915,6 +3319,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2969,6 +3376,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3023,6 +3433,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_are_also_used(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3079,6 +3492,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_can_be_partially_created_if_error_occurs(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3394,6 +3810,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     driver.configure(Fluent::Config::Element.new(
                        'ROOT', '', {
                          '@type' => 'elasticsearch',
+                         'bulk_message_request_threshold' => 20 * 1024 * 1024,
                        }, [
                          Fluent::Config::Element.new('buffer', 'tag', {
                                                        'chunk_keys' => ['tag', 'time'],
@@ -4344,7 +4761,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     stub_elastic
 
     ts = "2001/02/03 13:14:01,673+02:00"
-    index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
+    index = "logstash-#{Time.now.getutc.strftime("%Y.%m.%d")}"
 
     flexmock(driver.instance.router).should_receive(:emit_error_event)
       .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once