fluent-plugin-elasticsearch 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12f88bef6f61ff3acbbd654c63a6402f7374678e78cf438f2026551246e9e0fc
-  data.tar.gz: 7c8ffbd9626bf8204820326ed7a3bb9157f0e3e6b406dbba51f2cb66c899b6ca
+  metadata.gz: 46901e5f885a46c37ea29dca2968d53fd4026dec637f007936cabdd3ee070c5c
+  data.tar.gz: 21f3d3e9dd22752bcab060ea46ba12e8309147c2ddbb3be4aea656b527aff103
 SHA512:
-  metadata.gz: 1bc665321d8aed75d579a638d11d9254f2f3b50a4b2ce16b03e2db0ad2742fce155352c792cdeed1b244945430598d170eae546ab1e0cd4d4beedf5f0d9ea2c2
-  data.tar.gz: 6f54a08d03c458e7cf3dced5df532fbb5f5e2df2c9e8669c52a9bb3ebb0da231246717384653add9e585cbd2a068c00ab5be0fd12505fd04fccb7d0ca970faa3
+  metadata.gz: ed30b7f203f9c102fdedb0a28c88210f172e9bccab3f153de81ba88b821a065e1cfdb8caccc097fbb83745069c7c5cd052e70879501ebd83b472f55c835b6116
+  data.tar.gz: 3ded3c8f6041224546a71df34878148a14b0f2e47970b86efb11d8b4fe4008764680534149b1c56c1651bc7dbfffc0df26ad7a459c87e739c91f391d8da83f42

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,25 @@
 
 ### [Unreleased]
 
+### 4.2.0
+- ci: Add Ruby 2.7 jobs (#812)
+- Support Elasticsearch new style template (#810)
+
+### 4.1.4
+- ElasticsearchGenID update docs for hash_type (#809)
+- Handle api key header (#808)
+- index_template: Handle error object entirely on index creation failure (#807)
+
+### 4.1.3
+- Load multiple templates even if template_name and template_file given (#799)
+- Handle elasticsearch-ruby 7.9.0 using HTTP method changes (#795)
+
+### 4.1.2
+- Use Hash#dig instead of Hash#[] on retrieving version information from Elasticsearch info API (#793)
+
+### 4.1.1
+- Correct ILM explain on logstash_format case (#786)
+
 ### 4.1.0
 - Implement Fallback selector and configurable selector class (#782)
 

data/README.ElasticsearchGenID.md

@@ -26,7 +26,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   use_entire_record false
   record_keys []
   separator _
-  hash_type md5
+  hash_type sha1
 </match>
 ```
 
@@ -95,7 +95,7 @@ You can specify separator charactor to creating seed for hash generation.
 hash_type sha1
 ```
 
-You can specify hash algorithm.
+You can specify hash algorithm. Support algorithms `md5`, `sha1`, `sha256`, `sha512`. Default: `sha1`
 
 ## Advanced Usage
 

data/README.md

@@ -85,6 +85,7 @@ Current maintainers: @cosmo0920
   + [verify_es version at startup](#verify_es_version_at_startup)
   + [default_elasticsearch_version](#default_elasticsearch_version)
   + [custom_headers](#custom_headers)
+  + [api_key](#api_key)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
@@ -100,6 +101,7 @@ Current maintainers: @cosmo0920
   + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
+  + [use_legacy_template](#use_legacy_template)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -121,10 +123,12 @@ Current maintainers: @cosmo0920
 
 ## Requirements
 
-| fluent-plugin-elasticsearch  | fluentd | ruby |
-|-------------------|---------|------|
-| >= 2.0.0 | >= v0.14.20 | >= 2.1 |
-|  < 2.0.0 | >= v0.12.0 | >= 1.9 |
+| fluent-plugin-elasticsearch  | fluentd     | ruby   |
+|:----------------------------:|:-----------:|:------:|
+| >= 4.0.1                     | >= v0.14.22 | >= 2.3 |
+| >= 3.2.4 && < 4.0.1          | >= v0.14.22 | >= 2.1 |
+| >= 2.0.0 && < 3.2.3          | >= v0.14.20 | >= 2.1 |
+|  < 2.0.0                     | >= v0.12.0  | >= 1.9 |
 
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
@@ -429,7 +433,7 @@ Specify index templates in form of hash. Can contain multiple templates.
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}
 ```
 
-If `template_file` and `template_name` are set, then this parameter will be ignored.
+**Note:** Before ES plugin v4.1.2, if `template_file` and `template_name` are set, then this parameter will be ignored. In 4.1.3 or later, `template_file` and `template_name` can work with `templates`.
 
 ### customize_template
 
@@ -479,6 +483,8 @@ deflector_alias test-current
 
 If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
 
+**NOTE:** Since 4.1.1, `deflector_alias` is prohibited to use with `enable_ilm`.
+
 ### index_prefix
 
 This parameter is marked as obsoleted.
@@ -988,9 +994,28 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
 reload_after 100
 ```
 
+#### Tips
+
+The included sniffer class is not required `out_elasticsearch`.
+You should tell Fluentd where the sniffer class exists.
+
+If you use td-agent, you must put the following lines into `TD_AGENT_DEFAULT` file:
+
+```
+sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
+TD_AGENT_OPTIONS="--use-v1-config -r $sniffer"
+```
+
+If you use Fluentd directly, you must pass the following lines as Fluentd command line option:
+
+```
+sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
+$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
+```
+
 ### Selector Class Name
 
-The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should round robin and random selector cases. This doesn't work well when Fluentd should fallback behavior.
+The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should behave round robin and random selector cases. This doesn't work well when Fluentd should behave fallbacking from exhausted ES cluster to normal ES cluster.
 The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
 
 The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
@@ -1002,21 +1027,22 @@ selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
 
 #### Tips
 
-The included sniffer class does not required `out_elasticsearch`.
-You should tell Fluentd where the sniffer class exists.
+The included selector class is required in `out_elasticsearch` by default.
+But, your custom selector class is not required in `out_elasticsearch`.
+You should tell Fluentd where the selector class exists.
 
 If you use td-agent, you must put the following lines into `TD_AGENT_DEFAULT` file:
 
 ```
-sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
-TD_AGENT_OPTIONS="--use-v1-config -r $sniffer"
+selector=/path/to/your_awesome_selector.rb
+TD_AGENT_OPTIONS="--use-v1-config -r $selector"
 ```
 
 If you use Fluentd directly, you must pass the following lines as Fluentd command line option:
 
 ```
-sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
-$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
+selector=/path/to/your_awesome_selector.rb
+$ fluentd -r $selector [AND YOUR OTHER OPTIONS]
 ```
 
 ### Reload After
@@ -1082,6 +1108,14 @@ This parameter adds additional headers to request. The default value is `{}`.
 custom_headers {"token":"secret"}
 ```
 
+### api_key
+
+This parameter adds authentication header. The default value is `nil`.
+
+```
+api_key "ElasticsearchAPIKEY"
+```
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.
@@ -1270,6 +1304,18 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
 
 Default value is `nil`.
 
+## use_legacy_template
+
+Use legacy template or not.
+
+Elasticsearch 7.8 or later supports the brand new composable templates.
+
+For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+
+Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
+
+Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
@@ -1733,7 +1779,7 @@ ILM target index alias is created with `index_name` or an index which is calcula
 
 From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
 
-When using `deflector_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
+**NOTE:** Before Elasticsearch plugin v4.1.0, using `deflector_alias` parameter when ILM is enabled is permitted and handled, but, in the later releases such that 4.1.1 or later, it cannot use with when ILM is enabled.
 
 And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
 Because ILM settings are injected into their Elasticsearch templates.
@@ -1746,7 +1792,13 @@ It usually should be used with default value which is `default`.
 
 Then, ILM parameters are used in alias index like as:
 
-`<index_name/logstash_prefix><index_separator><application_name>-000001`.
+##### Simple `index_name` case:
+
+`<index_name><index_separator><application_name>-000001`.
+
+##### `logstash_format` as `true` case:
+
+`<logstash_prefix><logstash_prefix_separator><application_name><logstash_prefix_separator><logstash_dateformat>-000001`.
 
 #### Example ILM settings
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.1.0'
+  s.version       = '4.2.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -22,7 +22,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_exists?(name, host = nil)
-    client(host).indices.get_template(:name => name)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
     return true
   rescue Elasticsearch::Transport::Transport::Errors::NotFound
     return false
@@ -52,7 +56,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_put(name, template, host = nil)
-    client(host).indices.put_template(:name => name, :body => template)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
   end
 
   def indexcreation(index_name, host = nil)
@@ -61,15 +69,16 @@ module Fluent::ElasticsearchIndexTemplate
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
-      log.error("Error while index creation - #{index_name}: #{e.inspect}")
+      log.error("Error while index creation - #{index_name}", error: e)
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
                                                                 ilm_policy_id,
                                                                 get_template(template_file)) :
                      get_template(template_file), host)
@@ -80,6 +89,7 @@ module Fluent::ElasticsearchIndexTemplate
     if !template_exists?(inject_template_name, host)
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
                                                                 ilm_policy_id,
                                                                 get_template(template_file)) :
                      get_template(template_file), host)
@@ -89,10 +99,12 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
-                        inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id,
+                        inject_ilm_settings_to_template(deflector_alias_name,
+                                                        target_index,
+                                                        ilm_policy_id,
                                                         get_custom_template(template_file,
                                                                             customize_template))
                       else
@@ -115,26 +127,46 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template.delete('template') if template.include?('template')
-    template['index_patterns'] = "#{deflector_alias_name}-*"
-    template['order'] = template['order'] ? template['order'] + deflector_alias_name.split('-').length : 50 + deflector_alias_name.split('-').length
-    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    template['index_patterns'] = "#{target_index}-*"
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias_name})
     template
   end
 
-  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
+  def create_rollover_alias(target_index, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
      # ILM request to create alias.
     if rollover_index || enable_ilm
       if !client.indices.exists_alias(:name => deflector_alias_name)
-        if index_date_pattern.empty?
-          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
+        if @logstash_format
+          index_name_temp = '<'+target_index+'-000001>'
         else
-          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+          if index_date_pattern.empty?
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-000001>'
+          else
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+          end
         end
         indexcreation(index_name_temp, host)
         body = rollover_alias_payload(deflector_alias_name)

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -8,6 +8,7 @@ rescue LoadError
 end
 require 'json'
 require 'uri'
+require 'base64'
 begin
   require 'strptime'
 rescue LoadError
@@ -51,12 +52,13 @@ module Fluent::Plugin
       end
     end
 
-    RequestInfo = Struct.new(:host, :index, :ilm_index)
+    RequestInfo = Struct.new(:host, :index, :ilm_index, :ilm_alias)
 
     attr_reader :alias_indexes
     attr_reader :template_names
     attr_reader :ssl_version_options
     attr_reader :compressable_connection
+    attr_reader :api_key_header
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -155,6 +157,7 @@ EOC
     config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
     config_param :log_es_400_reason, :bool, :default => false
     config_param :custom_headers, :hash, :default => {}
+    config_param :api_key, :string, :default => nil, :secret => true
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
@@ -166,6 +169,7 @@ EOC
     config_param :ilm_policies, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
+    config_param :use_legacy_template, :bool, :default => true
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -212,6 +216,8 @@ EOC
         @remove_keys_on_update = @remove_keys_on_update.split ','
       end
 
+      @api_key_header = setup_api_key
+
       raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
       raise Fluent::ConfigError, "'max_retry_get_es_version' must be greater than or equal to zero." if @max_retry_get_es_version < 0
 
@@ -234,7 +240,7 @@ EOC
       if !dry_run?
         if @template_name && @template_file
           if @enable_ilm
-            raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
+            raise Fluent::ConfigError, "deflector_alias is prohibited to use with enable_ilm at same time." if @deflector_alias
           end
           if @ilm_policy.empty? && @ilm_policy_overwrite
             raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
@@ -247,7 +253,8 @@ EOC
             template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
-        elsif @templates
+        end 
+        if @templates
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
           end
@@ -397,6 +404,12 @@ EOC
       end
     end
 
+    def setup_api_key
+      return {} unless @api_key
+
+      { "Authorization" => "ApiKey " + Base64.strict_encode64(@api_key) }
+    end
+
     def dry_run?
       if Fluent::Engine.respond_to?(:dry_run_mode)
         Fluent::Engine.dry_run_mode
@@ -461,7 +474,10 @@ EOC
 
     def detect_es_major_version
       @_es_info ||= client.info
-      @_es_info["version"]["number"].to_i
+      unless version = @_es_info.dig("version", "number")
+        version = @default_elasticsearch_version
+      end
+      version.to_i
     end
 
     def client_library_version
@@ -554,7 +570,10 @@ EOC
                        else
                          {}
                        end
-        headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
+        headers = { 'Content-Type' => @content_type.to_s }
+                    .merge(@custom_headers)
+                    .merge(@api_key_header)
+                    .merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
@@ -771,9 +790,9 @@ EOC
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
-                   RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
+                   RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
-                   RequestInfo.new(host, nil, meta.delete("_alias".freeze))
+                   RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
                  end
 
           if split_request?(bulk_message, info)
@@ -944,16 +963,16 @@ EOC
 
     def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
-        if @alias_indexes.include? deflector_alias
+        if !@logstash_format && @alias_indexes.include?(deflector_alias)
           log.debug("Index alias #{deflector_alias} already exists (cached)")
-        elsif @template_names.include? template_name
+        elsif !@logstash_format && @template_names.include?(template_name)
           log.debug("Template name #{template_name} already exists (cached)")
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
             end
             ilm_policy = @ilm_policies[ilm_policy_id] || {}
             create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
@@ -967,11 +986,11 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
+      _logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
       if deflector_alias
         template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
       else
-        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
+        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? info.ilm_alias : index_name, ilm_policy_id, info.host)
       end
 
       begin