fluent-plugin-elasticsearch 3.8.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28cfb80e32dd25c9d67c582d4d39672515bbc9e0acb6a7b940ee1e36edd35de1
-  data.tar.gz: 6c2440335c4a1ebc03939dee6204cbd9fc6265797c9658d728df70fc63e9ba9d
+  metadata.gz: 4e963cd715f4b07ec74b88df45a4b27bd28ea3d9a81a986720eb103baf1f4ab8
+  data.tar.gz: 99adcede30be2bff2bce8eebdfb39e67a14028e46d073516411bfb81bcd623df
 SHA512:
-  metadata.gz: 7f67a2f892890ff80e34f6cd8aa14c5a61fc04095512f095d05554772eb384ddf54ec222006cf2b021ed08355d0af8575f0020671cd74de422c8b4c1e14989a0
-  data.tar.gz: 0d1919231ba8c8a88391a54d1853d992ba531f39c33826ff12ffd7258f9b3087ae5e18e1497edb0b6dac40998641a9ed79e28449cb90e2c9339a7a4e34406e4b
+  metadata.gz: a0fd61160246d393d0059737a2557ca595d943a5db15a554d12a7c20664b42972c2f466325a1c027cbddb1d3dcc0e6cb69fe3499f506aa95f17a47d809e6a7c9
+  data.tar.gz: 7031ebb1c3fbfcd8bd23662c77ea34b44512297f780f7be632765a30753df7802e677a66bf6d468c6d3c33314938add162a26a16a484e587d1fae416e56ca8b1

data/.travis.yml

@@ -2,12 +2,6 @@ language: ruby
 
 matrix:
   include:
-    - rvm: 2.1
-      gemfile: Gemfile
-    - rvm: 2.2
-      gemfile: Gemfile
-    - rvm: 2.3
-      gemfile: Gemfile
     - rvm: 2.4.6
       gemfile: Gemfile
     - rvm: 2.5.5
@@ -16,6 +10,8 @@ matrix:
       gemfile: gemfiles/Gemfile.ilm
     - rvm: 2.6.3
       gemfile: Gemfile
+    - rvm: 2.7.0
+      gemfile: Gemfile
 
 gemfile:
  - Gemfile

data/History.md

@@ -1,6 +1,13 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+
+### 4.0.0
+- Restructuring ILM related features (#701)
+- Extract placeholders in pipeline parameter (#695)
+- fix typo in `README.md` (#698)
+- Reduce log noise when not using rollover_index (#692)
+
 ### 3.8.0
 - Add FAQ for specifying index.codec (#679)
 - Add FAQ for connect_write timeout reached error (#687)

data/README.md

@@ -435,15 +435,14 @@ Specify this to override the index date pattern for creating a rollover index. T
 for example: <logstash-default-{now/d}-000001>. Overriding this changes the rollover time period. Setting
 "now/w{xxxx.ww}" would create weekly rollover indexes instead of daily.
 
-This setting only takes effect when combined with the [rollover_index](#rollover_index) setting.
+This setting only takes effect when combined with the [enable_ilm](#enable_ilm) setting.
 
-And rollover\_index is also used in Lifecycle Index Management(ILM) feature.
 ```
 index_date_pattern "now/w{xxxx.ww}" # defaults to "now/d"
 ```
 
-If empty string(`""`) is specified in `index_date_patter`, index date pattern is not used.
-Elasticsearch plugin just creates <`index_prefix`-`application_name`-000001> rollover index instead of <`index_prefix`-`application_name`-`{index_date_pattern}`-000001>.
+If empty string(`""`) is specified in `index_date_pattern`, index date pattern is not used.
+Elasticsearch plugin just creates <`target_index`-`application_name`-000001> rollover index instead of <`target_index`-`application_name`-`{index_date_pattern}`-000001>.
 
 If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
 
@@ -458,12 +457,9 @@ If [rollover_index](#rollover_index) is set, then this parameter will be in effe
 
 ### index_prefix
 
-Specify the index prefix for the rollover index to be created.
-```
-index_prefix mylogs # defaults to "logstash"
-```
-
-If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
+This parameter is marked as obsoleted.
+Consider to use [index_name](#index_name) for specify ILM target index when not using with logstash_format.
+When specifying `logstash_format` as true, consider to use [logstash_prefix](#logstash_prefix) to specify ILM target index prefix.
 
 ### application_name
 
@@ -472,7 +468,7 @@ Specify the application name for the rollover index to be created.
 application_name default # defaults to "default"
 ```
 
-If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
+If [enable_ilm](#enable_ilm is set, then this parameter will be in effect otherwise ignored.
 
 ### template_overwrite
 
@@ -1593,23 +1589,27 @@ Index lifecycle management is template based index management feature.
 
 Main ILM feature parameters are:
 
-* `rollover_index`
-* `deflector_alias`
+* `index_name` (when logstash_format as false)
+* `logstash_prefix` (when logstash_format as true)
 * `enable_ilm`
 * `ilm_policy_id`
 * `ilm_policy`
 
 They are not all mandatory parameters but they are used for ILM feature in effect.
 
+ILM target index alias is created with `index_name` or an index which is calculated from `logstash_prefix`.
+
+From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
+
+When using `deflectoe_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
+
 And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
 Because ILM settings are injected into their Elasticsearch templates.
 
+#### Example ILM settings
+
 ```aconf
 index_name fluentd-${tag}
-# Should specify rollover_index as true
-rollover_index true
-deflector_alias fluentd-${tag} # Should specify as same index_name
-index_prefix fluentd
 application_name ${tag}
 index_date_pattern "now/d"
 enable_ilm true
@@ -1618,11 +1618,44 @@ ilm_policy_id fluentd-policy
 # ilm_policy {} # Use default policy
 template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
-customize_template {"<<index_prefix>>": "fluentd"}
+# customize_template {"<<index_prefix>>": "fluentd"}
 ```
 
 Note: This plugin only creates rollover-enabled indices, which are aliases pointing to them and index templates, and creates an ILM policy if enabled.
 
+#### Create ILM indices in each day
+
+If you want to create new index in each day, you should use `logstash_format` style configuration:
+
+```aconf
+logstash_prefix fluentd
+application_name default
+index_date_pattern "now/d"
+enable_ilm true
+# Policy configurations
+ilm_policy_id fluentd-policy
+# ilm_policy {} # Use default policy
+template_name your-fluentd-template
+template_file /path/to/fluentd-template.json
+```
+
+#### Fixed ILM indices
+
+Also, users can use fixed ILM indices configuration.
+If `index_date_pattern` is set as `""`(empty string), Elasticsearch plugin won't attach date pattern in ILM indices:
+
+```aconf
+index_name fluentd
+application_name default
+index_date_pattern ""
+enable_ilm true
+# Policy configurations
+ilm_policy_id fluentd-policy
+# ilm_policy {} # Use default policy
+template_name your-fluentd-template
+template_file /path/to/fluentd-template.json
+```
+
 ### How to specify index codec
 
 Elasticsearch can handle compression methods for stored data such as LZ4 and best_compression.

data/appveyor.yml

@@ -1,8 +1,7 @@
 version: '{build}'
 install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - "%devkit%\\devkitvars.bat"
-  - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
+  - ridk.cmd enable
   - ruby --version
   - gem --version
   - bundle install
@@ -13,18 +12,9 @@ test_script:
 # https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
+    - ruby_version: "26-x64"
+    - ruby_version: "26"
     - ruby_version: "25-x64"
-      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "25"
-      devkit: C:\Ruby23\DevKit
     - ruby_version: "24-x64"
-      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "24"
-      devkit: C:\Ruby23\DevKit
-    - ruby_version: "23-x64"
-      devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "22-x64"
-      devkit: C:\Ruby23-x64\DevKit
-matrix:
-  allow_failures:
-    - ruby_version: "21"

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.8.0'
+  s.version       = '4.0.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake', '>= 0'
   s.add_development_dependency 'webmock', '~> 3'
-  s.add_development_dependency 'test-unit', '~> 3.1.0'
+  s.add_development_dependency 'test-unit', '~> 3.3.0'
   s.add_development_dependency 'minitest', '~> 5.8'
   s.add_development_dependency 'flexmock', '~> 2.0'
 end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -74,7 +74,7 @@ module Fluent::ElasticsearchIndexTemplate
                                                                 get_template(template_file)) :
                      get_template(template_file), host)
 
-      log.info("Template '#{inject_template_name}' overwritten with #{template_file}.")
+      log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
       return
     end
     if !template_exists?(inject_template_name, host)
@@ -128,7 +128,8 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
-    if rollover_index
+     # ILM request to create alias.
+    if rollover_index || enable_ilm
       if !client.indices.exists_alias(:name => deflector_alias_name)
         if index_date_pattern.empty?
           index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
@@ -152,7 +153,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.debug("The alias '#{deflector_alias_name}' is already present")
       end
     else
-      log.info("No index and alias creation action performed because rollover_index is set to '#{rollover_index}'")
+      log.debug("No index and alias creation action performed because rollover_index or enable_ilm is set to: '#{rollover_index}', '#{enable_ilm}'")
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -49,7 +49,7 @@ module Fluent::Plugin
       end
     end
 
-    RequestInfo = Struct.new(:host, :index)
+    RequestInfo = Struct.new(:host, :index, :ilm_index)
 
     attr_reader :alias_indexes
 
@@ -117,7 +117,8 @@ EOC
     config_param :index_date_pattern, :string, :default => "now/d"
     config_param :index_separator, :string, :default => "-"
     config_param :deflector_alias, :string, :default => nil
-    config_param :index_prefix, :string, :default => "logstash"
+    config_param :index_prefix, :string, :default => "logstash",
+                 obsoleted: "This parameter shouldn't be used in 4.0.0 or later. Specify ILM target index with using `index_name' w/o `logstash_format' or 'logstash_prefix' w/ `logstash_format' instead."
     config_param :application_name, :string, :default => "default"
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
@@ -150,7 +151,7 @@ EOC
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
     config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
-    config_param :compression_level, :enum, {list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression}
+    config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
@@ -173,8 +174,12 @@ EOC
       compat_parameters_convert(conf, :buffer)
 
       super
-      raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required." if not @buffer_config.chunk_keys.include? "tag" and not @buffer_config.chunk_keys.include? "_index"
-
+      if placeholder_substitution_needed_for_template?
+        # nop.
+      elsif not @buffer_config.chunk_keys.include? "tag" and
+        not @buffer_config.chunk_keys.include? "_index"
+        raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required."
+      end
       @time_parser = create_time_parser
       @backend_options = backend_options
 
@@ -209,11 +214,15 @@ EOC
       @alias_indexes = []
       if !dry_run?
         if @template_name && @template_file
-          if @rollover_index
-            raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
-          end
           if @enable_ilm
-            raise Fluent::ConfigError, "'rollover_index' and 'deflector_alias' must be provided if 'enable_ilm' is set true ." if !@deflector_alias &&!@deflector_alias
+            raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
+          end
+          if @logstash_format || placeholder_substitution_needed_for_template?
+            class << self
+              alias_method :template_installation, :template_installation_actual
+            end
+          else
+            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @application_name, @index_name)
           end
           verify_ilm_working if @enable_ilm
         elsif @templates
@@ -298,10 +307,6 @@ EOC
         end
       end
 
-      if @buffer_config.flush_thread_count < 2
-        log.warn "To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'."
-      end
-
       # Consider missing the prefix of "$." in nested key specifiers.
       @id_key = convert_compat_id_key(@id_key) if @id_key
       @parent_key = convert_compat_id_key(@parent_key) if @parent_key
@@ -356,7 +361,8 @@ EOC
       begin
         placeholder_validate!(name, param)
         true
-      rescue Fluent::ConfigError
+      rescue Fluent::ConfigError => e
+        log.debug("'#{name} #{param}' is tested built-in placeholder(s) but there is no valid placeholder(s). error: #{e}")
         false
       end
     end
@@ -651,7 +657,12 @@ EOC
       else
         application_name = nil
       end
-      return logstash_prefix, index_name, type_name, deflector_alias, application_name
+      if @pipeline
+        pipeline = extract_placeholders(@pipeline, chunk)
+      else
+        pipeline = nil
+      end
+      return logstash_prefix, index_name, type_name, deflector_alias, application_name, pipeline
     end
 
     def multi_workers_ready?
@@ -677,9 +688,9 @@ EOC
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
-                   RequestInfo.new(host, meta.delete("_index".freeze))
+                   RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
                  else
-                   RequestInfo.new(host, nil)
+                   RequestInfo.new(host, nil, meta["_index".freeze])
                  end
 
           if split_request?(bulk_message, info)
@@ -725,7 +736,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, index_name, type_name, deflector_alias, application_name = extracted_values
+      logstash_prefix, index_name, type_name, _deflector_alias, _application_name, pipeline = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -793,7 +804,7 @@ EOC
       meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
 
       if @pipeline
-        meta["pipeline".freeze] = @pipeline
+        meta["pipeline".freeze] = pipeline
       end
 
       @meta_config_map.each do |record_accessor, meta_key|
@@ -825,25 +836,45 @@ EOC
       wio.string
     end
 
-    # send_bulk given a specific bulk request, the original tag,
-    # chunk, and bulk_message_count
-    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, index_name, type_name, deflector_alias, application_name = extracted_values
+    def placeholder_substitution_needed_for_template?
+      placeholder?(:host, @host.to_s) ||
+        placeholder?(:index_name, @index_name.to_s) ||
+        placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
+        placeholder?(:deflector_alias, @deflector_alias.to_s) ||
+        placeholder?(:application_name, @application_name.to_s)
+    end
+
+    def template_installation(deflector_alias, application_name, target_index, host)
+      # for safety.
+    end
+
+    def template_installation_actual(deflector_alias, application_name, target_index, host=nil)
       if @template_name && @template_file
         if @alias_indexes.include? deflector_alias
           log.debug("Index alias #{deflector_alias} already exists (cached)")
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if @customize_template
-              template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, info.host)
+              template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
             else
-              template_install(@template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, info.host)
+              template_install(@template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
             end
-            create_rollover_alias(@index_prefix, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, info.host)
+            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
         end
       end
+    end
+
+    # send_bulk given a specific bulk request, the original tag,
+    # chunk, and bulk_message_count
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
+      logstash_prefix, index_name, _type_name, deflector_alias, application_name, _pipeline = extracted_values
+      if deflector_alias
+        template_installation(deflector_alias, application_name, index_name, info.host)
+      else
+        template_installation(info.ilm_index, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
+      end
 
       begin
 

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -246,7 +246,7 @@ module Fluent::Plugin
       # check for '${ ... }'
       #   yes => `eval`
       #   no  => return param
-      return param if (param =~ /\${.+}/).nil?
+      return param if (param.to_s =~ /\${.+}/).nil?
 
       # check for 'tag_parts[]'
         # separated by a delimiter (default '.')

data/test/plugin/test_out_elasticsearch.rb

@@ -307,7 +307,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
-  test 'invalid configuration of index lifecycle management' do
+  test 'valid configuration of index lifecycle management' do
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
 
@@ -316,7 +316,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
       template_name logstash
       template_file #{template_file}
     }
-    assert_raise(Fluent::ConfigError) {
+    stub_request(:get, "http://localhost:9200/_template/fluentd").
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:head, "http://localhost:9200/_alias/fluentd").
+      to_return(status: 404, body: "", headers: {})
+    stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/_alias/fluentd").
+      with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
+      to_return(status: 200, body: "", headers: {})
+    stub_request(:get, "http://localhost:9200/_xpack").
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}',
+                :headers => {"Content-Type"=> "application/json"})
+    stub_request(:get, "http://localhost:9200/_ilm/policy/logstash-policy").
+      to_return(status: 404, body: "", headers: {})
+    stub_request(:put, "http://localhost:9200/_ilm/policy/logstash-policy").
+      with(body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+      to_return(status: 200, body: "", headers: {})
+
+    assert_nothing_raised {
       driver(config)
     }
   end
@@ -379,6 +397,71 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "placeholder substitution needed?" do
+    data("host placeholder" => ["host", "host-${tag}.google.com"],
+         "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${tag}"],
+         "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${tag}"],
+         "application_name_placeholder" => ["application_name", "fluentd-${tag}"],
+        )
+    test 'tag placeholder' do |data|
+      param, value = data
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'elasticsearch',
+          param => value
+        }, [
+          Fluent::Config::Element.new('buffer', 'tag', {}, [])
+        ])
+      driver(config)
+
+      assert_true driver.instance.placeholder_substitution_needed_for_template?
+    end
+
+
+    data("host placeholder" => ["host", "host-%Y%m%d.google.com"],
+         "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-%Y%m%d"],
+         "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
+         "application_name_placeholder" => ["application_name", "fluentd-%Y%m%d"],
+        )
+    test 'time placeholder' do |data|
+      param, value = data
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'elasticsearch',
+          param => value
+        }, [
+          Fluent::Config::Element.new('buffer', 'time', {
+                                        'timekey' => '1d'
+                                      }, [])
+        ])
+      driver(config)
+
+      assert_true driver.instance.placeholder_substitution_needed_for_template?
+    end
+
+    data("host placeholder" => ["host", "host-${mykey}.google.com"],
+         "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
+         "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
+         "application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
+        )
+    test 'custom placeholder' do |data|
+      param, value = data
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'elasticsearch',
+          param => value
+        }, [
+          Fluent::Config::Element.new('buffer', 'mykey', {
+                                        'chunk_keys' => 'mykey',
+                                        'timekey' => '1d',
+                                      }, [])
+        ])
+      driver(config)
+
+      assert_true driver.instance.placeholder_substitution_needed_for_template?
+    end
+  end
+
   sub_test_case 'chunk_keys requirement' do
     test 'tag in chunk_keys' do
       assert_nothing_raised do
@@ -462,27 +545,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'connection exceptions' do
-    test 'default connection exception' do
-      driver(Fluent::Config::Element.new(
-               'ROOT', '', {
-                 '@type' => 'elasticsearch',
-                 'host' => 'log.google.com',
-                 'port' => 777,
-                 'scheme' => 'https',
-                 'path' => '/es/',
-                 'user' => 'john',
-                 'password' => 'doe',
-               }, [
-                 Fluent::Config::Element.new('buffer', 'tag', {
-                                             }, [])
-               ]
-             ))
-      logs = driver.logs
-      assert_logs_include(logs, /you should specify 2 or more 'flush_thread_count'/, 1)
-    end
-  end
-
   class GetElasticsearchVersionTest < self
     def create_driver(conf='', client_version="\"5.0\"")
       # For request stub to detect compatibility.
@@ -584,10 +646,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     driver(config)
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
+
     assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
   end
 
@@ -600,7 +659,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       end
     end
 
-
     def test_template_create_with_rollover_index_and_default_ilm
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
@@ -614,9 +672,74 @@ class ElasticsearchOutput < Test::Unit::TestCase
         password        doe
         template_name   logstash
         template_file   #{template_file}
-        rollover_index  true
+        index_date_pattern now/w{xxxx.ww}
+        index_name      logstash
+        enable_ilm      true
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
+    end
+
+    def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   logstash
+        template_file   #{template_file}
         index_date_pattern now/w{xxxx.ww}
         deflector_alias myapp_deflector
+        index_name      logstash
         enable_ilm      true
       }
 
@@ -664,12 +787,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       driver(config)
 
-      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
-
-      assert_requested(elastic_request)
+      assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
     end
 
     def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
@@ -685,9 +803,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
         password        doe
         template_name   logstash
         template_file   #{template_file}
-        rollover_index  true
         index_date_pattern ""
-        deflector_alias myapp_deflector
+        index_name      logstash
         enable_ilm      true
       }
 
@@ -704,23 +821,23 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
       stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
         with(basic_auth: ['john', 'doe'],
-             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/logstash").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -735,12 +852,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       driver(config)
 
-      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
-
-      assert_requested(elastic_request)
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
     end
 
     def test_template_create_with_rollover_index_and_custom_ilm
@@ -756,11 +868,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
         password        doe
         template_name   logstash
         template_file   #{template_file}
-        rollover_index  true
         index_date_pattern now/w{xxxx.ww}
-        deflector_alias myapp_deflector
         ilm_policy_id   fluentd-policy
         enable_ilm      true
+        index_name      logstash
         ilm_policy      {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
       }
 
@@ -777,24 +888,23 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
         with(basic_auth: ['john', 'doe'],
-             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
       stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
-        with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
-        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash").
+        with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
+        to_return(status: 200, body: "", headers: {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
@@ -808,12 +918,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       driver(config)
 
-      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
-
-      assert_requested(elastic_request)
+      assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
     end
 
     def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
@@ -829,10 +934,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
         password        doe
         template_name   logstash
         template_file   #{template_file}
-        rollover_index  true
         index_date_pattern now/w{xxxx.ww}
-        index_name fluentd-${tag}
-        deflector_alias myapp_deflector-${tag}
+        index_name logstash-${tag}
         enable_ilm      true
       }
 
@@ -849,23 +952,23 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-test").
+      stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-test").
+      stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test").
         with(basic_auth: ['john', 'doe'],
-             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-test-*\",\"order\":52}").
+             body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/logstash-test").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"myapp_deflector-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"logstash-test\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -884,9 +987,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
       end
-      assert_equal('fluentd-test', index_cmds.first['index']['_index'])
+      assert_equal('logstash-test', index_cmds.first['index']['_index'])
 
-      assert_equal ["myapp_deflector-test"], driver.instance.alias_indexes
+      assert_equal ["logstash-test"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -923,11 +1026,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
@@ -973,6 +1071,56 @@ class ElasticsearchOutput < Test::Unit::TestCase
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_alias_template.json')
 
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp_alias_template
+      template_file   #{template_file}
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      rollover_index  true
+      index_date_pattern now/w{xxxx.ww}
+      index_name    mylogs
+      application_name myapp
+    }
+
+    # connection start
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation of index which can rollover
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
+
+    def test_custom_template_with_rollover_index_create_and_deflector_alias
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_alias_template.json')
+
     config = %{
       host            logs.google.com
       port            777
@@ -986,7 +1134,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       rollover_index  true
       index_date_pattern now/w{xxxx.ww}
       deflector_alias myapp_deflector
-      index_prefix    mylogs
+      index_name    mylogs
       application_name myapp
     }
 
@@ -1017,12 +1165,63 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
+    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+  end
+
+  def test_custom_template_with_rollover_index_create_with_logstash_format
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_alias_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   myapp_alias_template
+      template_file   #{template_file}
+      customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
+      rollover_index  true
+      index_date_pattern now/w{xxxx.ww}
+      logstash_format true
+      logstash_prefix mylogs
+      application_name myapp
+    }
+
+    # connection start
+    stub_request(:head, "https://logs.google.com:777/es//").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # creation of index which can rollover
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if alias exists
+    timestr = Time.now.strftime("%Y.%m.%d")
+    stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 404, :body => "", :headers => {})
+    # put the alias for the index
+    stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs-#{timestr}").
+      with(basic_auth: ['john', 'doe']).
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver(config)
+
+    elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
+    driver.run(default_tag: 'custom-test') do
       driver.feed(sample_record)
     end
-
-    assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
+    assert_requested(elastic_request)
   end
 
   class CustomTemplateIndexLifecycleManagementTest < self
@@ -1048,10 +1247,78 @@ class ElasticsearchOutput < Test::Unit::TestCase
         template_name   myapp_alias_template
         template_file   #{template_file}
         customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
-        rollover_index  true
+        index_date_pattern now/w{xxxx.ww}
+        index_name    mylogs
+        application_name myapp
+        ilm_policy_id   fluentd-policy
+        enable_ilm      true
+      }
+
+      # connection start
+      stub_request(:head, "https://logs.google.com:777/es//").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if template exists
+      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      # creation
+      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # creation of index which can rollover
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => "", :headers => {})
+      # check if alias exists
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
+        with(basic_auth: ['john', 'doe']).
+        to_return(status: 404, body: "", headers: {})
+      stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
+        with(basic_auth: ['john', 'doe'],
+             body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
+        to_return(status: 200, body: "", headers: {})
+      # put the alias for the index
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "https://logs.google.com:777/es//_xpack").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe']).
+        to_return(:status => 404, :body => "", :headers => {})
+      stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
+        with(basic_auth: ['john', 'doe'],
+             :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
+        to_return(:status => 200, :body => "", :headers => {})
+
+      driver(config)
+
+      assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
+    end
+
+    def test_custom_template_with_rollover_index_create_and_default_ilm_with_deflector_alias
+      cwd = File.dirname(__FILE__)
+      template_file = File.join(cwd, 'test_alias_template.json')
+
+      config = %{
+        host            logs.google.com
+        port            777
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        template_name   myapp_alias_template
+        template_file   #{template_file}
+        customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
         index_date_pattern now/w{xxxx.ww}
         deflector_alias myapp_deflector
-        index_prefix    mylogs
+        index_name    mylogs
         application_name myapp
         ilm_policy_id   fluentd-policy
         enable_ilm      true
@@ -1102,12 +1369,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       driver(config)
 
-      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
-
-      assert_requested(elastic_request)
+      assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
     end
 
     def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
@@ -1124,11 +1386,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
         template_name   myapp_alias_template
         template_file   #{template_file}
         customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
-        rollover_index  true
         index_date_pattern now/w{xxxx.ww}
-        index_name fluentd-${tag}
-        deflector_alias myapp_deflector-${tag}
-        index_prefix    mylogs
+        index_name mylogs-${tag}
         application_name myapp
         ilm_policy_id   fluentd-policy
         enable_ilm      true
@@ -1147,24 +1406,24 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # creation of index which can rollover
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-custom-test").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
+      stub_request(:get, "https://logs.google.com:777/es//_template/mylogs-custom-test").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
+      stub_request(:put, "https://logs.google.com:777/es//_template/mylogs-custom-test").
         with(basic_auth: ['john', 'doe'],
-             body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-custom-test-*\"}").
+             body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-custom-test").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs-custom-test").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"myapp_deflector-custom-test\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"mylogs-custom-test\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -1183,9 +1442,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
       driver.run(default_tag: 'custom-test') do
         driver.feed(sample_record)
       end
-      assert_equal('fluentd-custom-test', index_cmds.first['index']['_index'])
+      assert_equal('mylogs-custom-test', index_cmds.first['index']['_index'])
 
-      assert_equal ["myapp_deflector-custom-test"], driver.instance.alias_indexes
+      assert_equal ["mylogs-custom-test"], driver.instance.alias_indexes
 
       assert_requested(elastic_request)
     end
@@ -1204,10 +1463,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
         template_name   myapp_alias_template
         template_file   #{template_file}
         customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
-        rollover_index  true
         index_date_pattern now/w{xxxx.ww}
-        deflector_alias myapp_deflector
-        index_prefix    mylogs
+        index_name    mylogs
         application_name myapp
         ilm_policy_id   fluentd-policy
         enable_ilm      true
@@ -1231,19 +1488,19 @@ class ElasticsearchOutput < Test::Unit::TestCase
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 200, :body => "", :headers => {})
       # check if alias exists
-      stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
+      stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
         with(basic_auth: ['john', 'doe']).
         to_return(:status => 404, :body => "", :headers => {})
-      stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
+      stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 404, body: "", headers: {})
-      stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
         with(basic_auth: ['john', 'doe']).
         to_return(status: 200, body: "", headers: {})
       # put the alias for the index
-      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
+      stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/mylogs").
         with(basic_auth: ['john', 'doe'],
-             :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
+             :body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, "https://logs.google.com:777/es//_xpack").
         with(basic_auth: ['john', 'doe']).
@@ -1258,12 +1515,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
       driver(config)
 
-      elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
-
-      assert_requested(elastic_request)
+      assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
     end
   end
 
@@ -1298,11 +1550,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
   end
 
@@ -1338,11 +1585,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
@@ -1363,7 +1605,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
       deflector_alias myapp_deflector
       rollover_index  true
-      index_prefix    mylogs
+      index_name    mylogs
       application_name myapp
     }
 
@@ -1394,11 +1636,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
   end
 
@@ -1423,13 +1660,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
       with(basic_auth: ['john', 'doe']).
       to_return(:status => 404, :body => "", :headers => {})
 
-    driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
     assert_raise(RuntimeError) {
-      driver.run(default_tag: 'test') do
-        driver.feed(sample_record)
-      end
+      driver(config)
     }
   end
 
@@ -1497,11 +1730,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
       raise Faraday::ConnectionFailed, "Test message"
     end
 
-    driver(config)
-
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
+    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
+      driver(config)
     end
 
     assert_equal(4, connection_resets)
@@ -1534,11 +1764,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:778/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_equal(4, connection_resets)
   end
 
@@ -1674,11 +1899,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     driver(config)
 
-    stub_elastic("https://logs.google.com:777/es//_bulk")
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-
     assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
 
     assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash1")
@@ -2123,6 +2343,58 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(pipeline, index_cmds.first['index']['pipeline'])
   end
 
+  class PipelinePlaceholdersTest < self
+    def test_writes_to_default_index_with_pipeline_tag_placeholder
+      pipeline = "fluentd-${tag}"
+      driver.configure("pipeline #{pipeline}")
+      stub_elastic
+      driver.run(default_tag: 'test.builtin.placeholder') do
+        driver.feed(sample_record)
+      end
+      assert_equal("fluentd-test.builtin.placeholder", index_cmds.first['index']['pipeline'])
+    end
+
+    def test_writes_to_default_index_with_pipeline_time_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'pipeline' => 'fluentd-%Y%m%d',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time', {
+                                                         'chunk_keys' => ['tag', 'time'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      pipeline = "fluentd-#{time.getutc.strftime("%Y%m%d")}"
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_equal(pipeline, index_cmds.first['index']['pipeline'])
+    end
+
+    def test_writes_to_default_index_with_pipeline_custom_key_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'pipeline' => 'fluentd-${pipeline_id}',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      pipeline_id = "mypipeline"
+      logstash_index = "fluentd-#{pipeline_id}"
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['pipeline'])
+    end
+  end
+
   def test_writes_to_target_index_key_fallack
     driver.configure("target_index_key @target_index\n")
     stub_elastic