fluent-plugin-elasticsearch 5.0.0 → 5.2.3

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -28,17 +28,33 @@ module Fluent::ElasticsearchIndexTemplate
       client(host).indices.get_index_template(:name => name)
     end
     return true
-  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+  rescue TRANSPORT_CLASS::Transport::Errors::NotFound
     return false
   end
 
+  def host_unreachable_exceptions
+    if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      # elasticsearch-ruby 8.0.0's elastic-transport uses
+      # direct callable #host_unreachable_exceptions again.
+      client.transport.host_unreachable_exceptions
+    elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
+      # Elasticsearch class on Transport.
+      # This is why #host_unreachable_exceptions is not callable directly
+      # via transport (not transport's transport instance accessor) any more.
+      client.transport.transport.host_unreachable_exceptions
+    else
+      client.transport.host_unreachable_exceptions
+    end
+  end
+
   def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
-    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
+    transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries
@@ -66,7 +82,7 @@ module Fluent::ElasticsearchIndexTemplate
 
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
-  rescue Elasticsearch::Transport::Transport::Error => e
+  rescue TRANSPORT_CLASS::Transport::Error => e
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -1,6 +1,7 @@
 require 'elasticsearch'
+require_relative 'elasticsearch_compat'
 
-class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
+class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
 
   def hosts
     @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -53,7 +53,7 @@ module Fluent::Plugin
       seed += tag + separator if @include_tag_in_seed
       seed += time.to_s + separator if @include_time_in_seed
       if @use_entire_record
-        record.each {|k,v| seed += "|#{k}|#{v}"}
+        record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
       else
         seed += record_keys.map {|k| record[k]}.join(separator)
       end

data/lib/fluent/plugin/in_elasticsearch.rb

@@ -3,6 +3,7 @@ require 'elasticsearch'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
 require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   class ElasticsearchInput < Input
@@ -218,7 +219,7 @@ module Fluent::Plugin
 
         headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
 
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
           connection_options.merge(
             options: {
               reload_connections: local_reload_connections,

data/lib/fluent/plugin/oj_serializer.rb

@@ -1,10 +1,11 @@
 require 'oj'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   module Serializer
 
     class Oj
-      include Elasticsearch::Transport::Transport::Serializer::Base
+      include TRANSPORT_CLASS::Transport::Serializer::Base
 
       # De-serialize a Hash from JSON string
       #

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,10 +2,7 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
-begin
-  require 'elasticsearch/xpack'
-rescue LoadError
-end
+require 'set'
 require 'json'
 require 'uri'
 require 'base64'
@@ -13,6 +10,7 @@ begin
   require 'strptime'
 rescue LoadError
 end
+require 'resolv'
 
 require 'fluent/plugin/output'
 require 'fluent/event'
@@ -21,6 +19,7 @@ require 'fluent/time'
 require 'fluent/unique_id'
 require 'fluent/log-ext'
 require 'zlib'
+require_relative 'elasticsearch_compat'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
@@ -70,7 +69,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
-    TARGET_BULK_BYTES = 20 * 1024 * 1024
+    DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
     config_param :host, :string,  :default => 'localhost'
@@ -164,7 +163,7 @@ EOC
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
-    config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
     config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
@@ -174,6 +173,7 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
     config_param :catch_transport_exception_on_retry, :bool, :default => true
+    config_param :target_index_affinity, :bool, :default => false
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -410,11 +410,11 @@ EOC
         end
       end
 
-      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+      if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
         if compression
           raise Fluent::ConfigError, <<-EOC
             Cannot use compression with elasticsearch-transport plugin version < 7.2.0
-            Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
+            Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
             Please consider to upgrade ES client.
           EOC
         end
@@ -492,7 +492,11 @@ EOC
     end
 
     def detect_es_major_version
-      @_es_info ||= client.info
+      begin
+        @_es_info ||= client.info
+      rescue ::Elasticsearch::UnsupportedProductError => e
+        raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+      end
       begin
         unless version = @_es_info.dig("version", "number")
           version = @default_elasticsearch_version
@@ -610,7 +614,7 @@ EOC
                     .merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: local_reload_connections,
                                                                               reload_on_failure: @reload_on_failure,
@@ -668,7 +672,11 @@ EOC
           end
         end.compact
       else
-        [{host: @host, port: @port, scheme: @scheme.to_s}]
+        if Resolv::IPv6::Regex.match(@host)
+          [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
+        else
+          [{host: @host, port: @port, scheme: @scheme.to_s}]
+        end
       end.each do |host|
         host.merge!(user: @user, password: @password) if !host[:user] && @user
         host.merge!(path: @path) if !host[:path] && @path
@@ -819,6 +827,7 @@ EOC
       bulk_message = Hash.new { |h,k| h[k] = '' }
       header = {}
       meta = {}
+      unpackedMsgArr = {}
 
       tag = chunk.metadata.tag
       chunk_id = dump_unique_id_hex(chunk.unique_id)
@@ -829,22 +838,27 @@ EOC
                extract_placeholders(@host, chunk)
              end
 
+      affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
         record = inject_chunk_id_to_record_if_needed(record, chunk_id)
 
         begin
-          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
           info = if @include_index_in_url
                    RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
                    RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
                  end
 
+          unpackedMsgArr[info] = [] if unpackedMsgArr[info].nil?
+          unpackedMsgArr[info] << {:time => time, :record => record}
+
           if split_request?(bulk_message, info)
             bulk_message.each do |info, msgs|
-              send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
+              send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+              unpackedMsgArr[info].clear
               msgs.clear
               # Clear bulk_message_count for this info.
               bulk_message_count[info] = 0;
@@ -867,11 +881,49 @@ EOC
       end
 
       bulk_message.each do |info, msgs|
-        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
+        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+
+        unpackedMsgArr[info].clear
         msgs.clear
       end
     end
 
+    def target_index_affinity_enabled?()
+      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
+    end
+
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      if target_index_affinity_enabled?()
+        id_key_accessor = record_accessor_create(@id_key)
+        ids = Set.new
+        chunk.msgpack_each do |time, record|
+          next unless record.is_a? Hash
+          begin
+            ids << id_key_accessor.call(record)
+          end
+        end
+        log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
+        options = {
+          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
+        }
+        query = {
+          'query' => { 'ids' => { 'values' => ids.to_a } },
+          '_source' => false,
+          'sort' => [
+            {"_index" => {"order" => "desc"}}
+         ]
+        }
+        result = client.search(options.merge(:body => Yajl.dump(query)))
+        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
+        result['hits']['hits'].each do |hit|
+          indices[hit["_id"]] = hit["_index"]
+          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
+        end
+      end
+      indices
+    end
+
     def split_request?(bulk_message, info)
       # For safety.
     end
@@ -884,7 +936,7 @@ EOC
       false
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
@@ -925,6 +977,15 @@ EOC
         record[@tag_key] = tag
       end
 
+      # If affinity target indices map has value for this particular id, use it as target_index
+      if !affinity_target_indices.empty?
+        id_accessor = record_accessor_create(@id_key)
+        id_value = id_accessor.call(record)
+        if affinity_target_indices.key?(id_value)
+          target_index = affinity_target_indices[id_value]
+        end
+      end
+
       target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)
@@ -934,12 +995,12 @@ EOC
         elsif @last_seen_major_version == 7
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
-        elsif @last_seen_major_version >=8
+        elsif @last_seen_major_version >= 8
           log.debug "Detected ES 8.x or above: document type will not be used."
           target_type = nil
         end
       else
-        if @suppress_type_name && @last_seen_major_version >= 7
+        if @suppress_type_name && @last_seen_major_version == 7
           target_type = nil
         elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
@@ -1036,7 +1097,7 @@ EOC
 
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
-    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info, unpacked_msg_arr)
       _logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
       if deflector_alias
         template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
@@ -1059,7 +1120,7 @@ EOC
 
         if response['errors']
           error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
-          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
         end
       rescue RetryStreamError => e
         log.trace "router.emit_stream for retry stream doing..."

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -1,3 +1,4 @@
+
 require_relative 'out_elasticsearch'
 
 module Fluent::Plugin
@@ -8,6 +9,11 @@ module Fluent::Plugin
     helpers :event_emitter
 
     config_param :data_stream_name, :string
+    config_param :data_stream_ilm_name, :string, :default => nil
+    config_param :data_stream_template_name, :string, :default => nil
+    config_param :data_stream_ilm_policy, :string, :default => nil
+    config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -17,134 +23,186 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      begin
-        require 'elasticsearch/xpack'
-      rescue LoadError
-        raise Fluent::ConfigError, "'elasticsearch/xpack'' is required for <@elasticsearch_data_stream>."
+      if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+        begin
+          require 'elasticsearch/api'
+          require 'elasticsearch/xpack'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
+        end
+      else
+        begin
+          require 'elasticsearch/api'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
+        end
       end
 
+      @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+      @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
+
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
-        validate_data_stream_name
+        validate_data_stream_parameters
       else
         @use_placeholder = true
         @data_stream_names = []
       end
 
-      @client = client
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name)
-          create_index_template(@data_stream_name)
-          create_data_stream(@data_stream_name)
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_data_stream(@data_stream_name)
+          end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
         end
       end
     end
 
-    def validate_data_stream_name
-      unless valid_data_stream_name?
-        unless start_with_valid_characters?
-          if not_dots?
-            raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
-          else
-            raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name,
+       "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
           end
-        end
-        unless valid_characters?
-          raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
-        end
-        unless lowercase_only?
-          raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
-        end
-        if @data_stream_name.bytes.size > 255
-          raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
         end
       end
     end
 
-    def create_ilm_policy(name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
+      unless @data_stream_ilm_policy_overwrite
+        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
+      end
+
       params = {
-        policy_id: "#{name}_policy",
-        body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
+        body: @data_stream_ilm_policy
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.xpack.ilm.put_policy(params)
+        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).enrich.put_policy(params.merge(name: ilm_name))
+        else
+          client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
+        end
       end
     end
 
-    def create_index_template(name)
+    def create_index_template(datastream_name, template_name, ilm_name, host = nil)
+      return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
       body = {
-        "index_patterns" => ["#{name}*"],
+        "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
         "template" => {
           "settings" => {
-            "index.lifecycle.name" => "#{name}_policy"
+            "index.lifecycle.name" => "#{ilm_name}"
           }
         }
       }
       params = {
-        name: name,
+        name: template_name,
         body: body
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.put_index_template(params)
+        client(host).indices.put_index_template(params)
       end
     end
 
-    def data_stream_exist?(name)
+    def data_stream_exist?(datastream_name, host = nil)
       params = {
-        "name": name
+        name: datastream_name
       }
       begin
-        response = @client.indices.get_data_stream(params)
-        return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+        response = client(host).indices.get_data_stream(params)
+        return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
+      rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
         log.info "Specified data stream does not exist. Will be created: <#{e}>"
         return false
       end
     end
 
-    def create_data_stream(name)
-      return if data_stream_exist?(name)
+    def create_data_stream(datastream_name, host = nil)
+      return if data_stream_exist?(datastream_name, host)
       params = {
-        "name": name
+        name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.create_data_stream(params)
+        client(host).indices.create_data_stream(params)
       end
     end
 
-    def valid_data_stream_name?
-      lowercase_only? and
-        valid_characters? and
-        start_with_valid_characters? and
-        not_dots? and
-        @data_stream_name.bytes.size <= 255
+    def ilm_policy_exists?(policy_id, host = nil)
+      begin
+        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).enrich.get_policy(name: policy_id)
+        else
+          client(host).ilm.get_policy(policy_id: policy_id)
+        end
+        true
+      rescue
+        false
+      end
     end
 
-    def lowercase_only?
-      @data_stream_name.downcase == @data_stream_name
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue TRANSPORT_CLASS::Transport::Errors::NotFound
+      return false
     end
 
-    def valid_characters?
-      not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
     end
 
-    def start_with_valid_characters?
-      not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
     end
 
-    def not_dots?
-      not (@data_stream_name == "." or @data_stream_name == "..")
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
+    end
+
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
+    end
+
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
     end
 
     def client_library_version
@@ -157,12 +215,18 @@ module Fluent::Plugin
 
     def write(chunk)
       data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      data_stream_ilm_name = @data_stream_ilm_name
+      host = nil
       if @use_placeholder
+        host = extract_placeholders(@host, chunk)
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
         unless @data_stream_names.include?(data_stream_name)
           begin
-            create_ilm_policy(data_stream_name)
-            create_index_template(data_stream_name)
+            create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
             create_data_stream(data_stream_name)
             @data_stream_names << data_stream_name
           rescue => e
@@ -179,8 +243,14 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+
         begin
-          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          unless record.has_key?("@timestamp")
+            record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          end
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
           router.emit_error_event(tag, time, record, e)
@@ -192,12 +262,12 @@ module Fluent::Plugin
         body: bulk_message
       }
       begin
-        response = @client.bulk(params)
+        response = client(host).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end
       rescue => e
-        log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
       end
     end