fluent-plugin-elasticsearch 4.0.5 → 4.0.10

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'elasticsearch'
 require 'fluent/plugin/elasticsearch_index_lifecycle_management'
 

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
+        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'date'
 require 'fluent/test/helpers'
 require 'json'
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
   end
 
+  test "invalid configuration" do
+    assert_raise(Fluent::ConfigError) do
+      create_driver("use_record_as_seed true")
+    end
+  end
+
   def sample_record
     {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
     assert_equal(Base64.strict_encode64(SecureRandom.uuid),
                  d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
   end
+
+  class UseRecordAsSeedTest < self
+    data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
+         "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
+         "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
+         "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
+        )
+    def test_simple(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
+         "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
+         "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
+         "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
+    def test_record_with_tag(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
+         "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
+         "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
+         "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
+    def test_record_with_time(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
+         "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
+         "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
+         "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+
+  class UseEntireRecordAsSeedTest < self
+    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
+         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
+         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
+         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    def test_record
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
+         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
+         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
+         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    def test_record_with_tag
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
+         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
+         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
+         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    def test_record_with_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
+         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
+         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
+         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
 end

data/test/plugin/test_in_elasticsearch.rb

@@ -4,6 +4,7 @@ require 'fluent/test/helpers'
 require 'json'
 require 'fluent/test/driver/input'
 require 'flexmock/test_unit'
+require 'fluent/plugin/in_elasticsearch'
 
 class ElasticsearchInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -16,7 +17,6 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/in_elasticsearch'
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)

data/test/plugin/test_out_elasticsearch.rb

@@ -1,11 +1,12 @@
-require 'helper'
+require_relative '../helper'
 require 'date'
 require 'fluent/test/helpers'
 require 'json'
 require 'fluent/test/driver/output'
 require 'flexmock/test_unit'
+require 'fluent/plugin/out_elasticsearch'
 
-class ElasticsearchOutput < Test::Unit::TestCase
+class ElasticsearchOutputTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
@@ -13,7 +14,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/out_elasticsearch'
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
@@ -230,7 +230,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil instance.ssl_max_version
     assert_nil instance.ssl_min_version
     if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
-      assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
+      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
                    instance.ssl_version_options)
     else
       assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
@@ -251,6 +251,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
     assert_false instance.compression
     assert_equal :no_compression, instance.compression_level
+    assert_true instance.http_backend_excon_nonblock
   end
 
   test 'configure compression' do
@@ -283,7 +284,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
   end
 
   test 'check compression option is passed to transport' do
@@ -294,7 +304,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
     instance = driver(config).instance
 
-    assert_equal true, instance.client.transport.options[:compression]
+    assert_equal false, instance.client.transport.options[:compression]
+
+    stub_request(:post, "http://localhost:9200/_bulk").
+      to_return(status: 200, body: "", headers: {})
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    compressable = instance.compressable_connection
+
+    assert_equal true, instance.client(nil, compressable).transport.options[:compression]
   end
 
   test 'configure Content-Type' do
@@ -332,6 +351,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  sub_test_case 'Check TLS handshake stuck warning log' do
+    test 'warning TLS log' do
+      config = %{
+        scheme https
+        http_backend_excon_nonblock false
+        ssl_version TLSv1_2
+        @log_level info
+      }
+      driver(config)
+      logs = driver.logs
+      assert_logs_include(logs, /TLS handshake will be stucked with block connection.\n                    Consider to set `http_backend_excon_nonblock` as true\n/)
+    end
+  end
+
   sub_test_case 'ILM default config' do
     setup do
       begin
@@ -535,6 +568,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "template_name_placeholder" => ["template_name", "logstash-${mykey}"],
          "customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
+         "logstash_dateformat_placeholder" => ["logstash_dateformat", "${mykey}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
          "application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
         )
@@ -560,6 +594,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
          "customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
+         "logstash_dataformat_placeholder" => ["logstash_dateformat", "${es_index}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
          "application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
         )
@@ -710,6 +745,60 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  class GetElasticsearchVersionWithFallbackTest < self
+    def create_driver(conf='', client_version="\"5.0\"")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      # Ensure original implementation existence.
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          @_es_info ||= client.info
+          @_es_info["version"]["number"].to_i
+        end
+      CODE
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+    end
+
+    data("Elasticsearch 5" => ["5.0", 5],
+         "Elasticsearch 6" => ["6.0", 6],
+         "Elasticsearch 7" => ["7.0", 7],
+         "Elasticsearch 8" => ["8.0", 8])
+    def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
+      client_version, es_major_version = data
+      config = %{
+        host            logs.google.com
+        port            778
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        verify_es_version_at_startup true
+        max_retry_get_es_version 2
+        fail_on_detecting_es_version_retry_exceed false
+        default_elasticsearch_version #{es_major_version}
+        @log_level info
+      }
+
+      connection_resets = 0
+      stub_request(:get, "https://logs.google.com:778/es//").
+        with(basic_auth: ['john', 'doe']) do |req|
+        connection_resets += 1
+        raise Faraday::ConnectionFailed, "Test message"
+      end
+
+      d = create_driver(config, client_version)
+
+      assert_equal es_major_version, d.instance.default_elasticsearch_version
+      assert_equal 3, connection_resets
+      assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
+    end
+  end
+
   def test_template_already_present
     config = %{
       host            logs.google.com
@@ -1228,6 +1317,224 @@ class ElasticsearchOutput < Test::Unit::TestCase
       assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
     end
 
+    def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
+        index_date_pattern now/w{xxxx.ww}
+        ilm_policy_id   fluentd-policy
+        enable_ilm      true
+        index_name      logstash
+        ilm_policies    {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
+        with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+        to_return(status: 200, body: "", headers: {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record)
+      end
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
+
+      assert_requested(elastic_request)
+    end
+
+    class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
+      def test_tag_placeholder
+        cwd = File.dirname(__FILE__)
+        template_file = File.join(cwd, 'test_template.json')
+
+        config = %{
+          host            logs.google.com
+          port            777
+          scheme          https
+          path            /es/
+          user            john
+          password        doe
+          template_name   logstash
+          template_file   #{template_file}
+          index_date_pattern now/w{xxxx.ww}
+          ilm_policy_id   ${tag}
+          enable_ilm      true
+          index_name      logstash
+          ilm_policies    {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
+        }
+
+        # connection start
+        stub_request(:head, "https://logs.google.com:777/es//").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        # check if template exists
+        stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        # creation
+        stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        # check if alias exists
+        stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(status: 404, body: "", headers: {})
+        stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
+          to_return(status: 200, body: "", headers: {})
+        # put the alias for the index
+        stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
+          with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+          to_return(status: 200, body: "", headers: {})
+        stub_request(:get, "https://logs.google.com:777/es//_xpack").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+        stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
+          to_return(:status => 200, :body => "", :headers => {})
+
+        driver(config)
+
+        elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+        driver.run(default_tag: 'fluentd-policy') do
+          driver.feed(sample_record)
+        end
+        assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
+
+        assert_requested(elastic_request)
+      end
+
+      def test_tag_placeholder_with_multiple_policies
+        cwd = File.dirname(__FILE__)
+        template_file = File.join(cwd, 'test_template.json')
+
+        config = %{
+          host            logs.google.com
+          port            777
+          scheme          https
+          path            /es/
+          user            john
+          password        doe
+          template_name   logstash
+          template_file   #{template_file}
+          index_date_pattern now/w{xxxx.ww}
+          ilm_policy_id   ${tag}
+          enable_ilm      true
+          index_name      logstash
+          ilm_policies    {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
+        }
+
+        # connection start
+        stub_request(:head, "https://logs.google.com:777/es//").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        # check if template exists
+        stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        # creation
+        stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        # check if alias exists
+        stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe']).
+          to_return(status: 404, body: "", headers: {})
+        stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
+          to_return(status: 200, body: "", headers: {})
+        # put the alias for the index
+        stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => "", :headers => {})
+        stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
+          with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+          to_return(status: 200, body: "", headers: {})
+        stub_request(:get, "https://logs.google.com:777/es//_xpack").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+        stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
+          with(basic_auth: ['john', 'doe']).
+          to_return(:status => 404, :body => "", :headers => {})
+        stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
+          with(basic_auth: ['john', 'doe'],
+               body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
+          to_return(:status => 200, :body => "", :headers => {})
+
+        driver(config)
+
+        elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+        driver.run(default_tag: 'fluentd-policy2') do
+          driver.feed(sample_record)
+        end
+        assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
+
+        assert_requested(elastic_request)
+      end
+    end
+
     def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
@@ -2840,6 +3147,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   data("border"        => {"es_version" => 6, "_type" => "mytype"},
        "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
       )
   def test_writes_to_speficied_type(data)
     driver('', data["es_version"]).configure("type_name mytype\n")
@@ -2852,6 +3160,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   data("border"        => {"es_version" => 6, "_type" => "mytype.test"},
        "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
       )
   def test_writes_to_speficied_type_with_placeholders(data)
     driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
@@ -2862,6 +3171,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(data['_type'], index_cmds.first['index']['_type'])
   end
 
+  data("border"        => {"es_version" => 6, "_type" => "mytype.test"},
+       "fixed_behavior"=> {"es_version" => 7, "_type" => nil},
+       "to be nil"     => {"es_version" => 8, "_type" => nil},
+      )
+  def test_writes_to_speficied_type_with_suppress_type_name(data)
+    driver('', data["es_version"])
+      .configure("type_name mytype.${tag}\nsuppress_type_name true")
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal(data['_type'], index_cmds.first['index']['_type'])
+  end
+
   data("old"           => {"es_version" => 2, "_type" => "local-override"},
        "old_behavior"  => {"es_version" => 5, "_type" => "local-override"},
        "border"        => {"es_version" => 6, "_type" => "fluentd"},