fluent-plugin-elasticsearch 5.0.3 → 5.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb418d889b91bf79c37c1cd72789981eb4133bcfc34e21b26e79bb919462272
-  data.tar.gz: 168ddc77fb73216da63f1ce65ed673cf12ab13d6db549fc3e2da81956c938990
+  metadata.gz: 27d74e7048671def02b98e337c052c395152021a4a3f4c2138d1780c725d09bd
+  data.tar.gz: eb5282b8e688b091700a549c711af2f1959bc9b1b2c4f6fd1b49e3119c62ddb7
 SHA512:
-  metadata.gz: 5c4a2a8f63b25ea8785e0d4ef291043a93a86fea8f42638caaf46dd3efe29fd0a12919dd339d8a28c97aef65494ba054a3ff3312a122ab2cd89cb54cb069055c
-  data.tar.gz: 6b13851ce29b2a6f2083ebc57ae733a9ca00d79a9fd09ac4c7630d99a9c6d2da302c48f2e0cd901810e0a851911e14f8355fc336e21afd97d3f33fe205b0c78a
+  metadata.gz: 3a3ad9fa5259fcd1e80a85bdf7d1acd11cd26675d4f47f326100db242f0f9320232530099eb5346e5ea11aba76c4cc66cfc2e97f6393ca95d1227281217283ea
+  data.tar.gz: f97182a9487be71d34ddcd8ec2dd046eca9c6de1cae55d3842feeeaef627f23a05862be4783aee37fe38c84cba3bf984a51fb7fb3e8bad50f1bf0f57c956803e

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '2.6', '2.7', '3.0' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -1,6 +1,25 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+
+### 5.1.1
+-  Report appropriate error for data_stream parameters (#922)
+- Add ILM and template parameters for data streams (#920)
+- Support Buffer in Data Stream Output (#917)
+
+### 5.1.0
+- Correct default target bytes value (#914)
+- Handle elasticsearch-ruby 7.14 properly (#913)
+
+### 5.0.5
+- Drop json_parse_exception messages for bulk failures (#900)
+- GitHub Actions: Drop Ruby 2.5 due to EOL (#894)
+
+### 5.0.4
+- test: out_elasticsearch: Remove a needless headers from affinity stub (#888)
+- Target Index Affinity (#883)
+
+### 5.0.3
 - Fix use_legacy_template documentation (#880)
 - Add FAQ for dynamic index/template (#878)
 - Handle IPv6 address string on host and hosts parameters (#877)

data/README.md

@@ -11,7 +11,7 @@ Send your logs to Elasticsearch (and search them with Kibana maybe?)
 
 Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service)
 
-Current maintainers: @cosmo0920
+Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920), [Kentaro Hayashi | @kenhys](https://github.com/kenhys)
 
 * [Installation](#installation)
 * [Usage](#usage)
@@ -38,6 +38,7 @@ Current maintainers: @cosmo0920
   + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
+  + [target_index_affinity](#target_index_affinity)
   + [template_name](#template_name)
   + [template_file](#template_file)
   + [template_overwrite](#template_overwrite)
@@ -454,6 +455,75 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
+### target_index_affinity
+
+Enable plugin to dynamically select logstash time based target index in update/upsert operations based on already indexed data rather than current time of indexing.
+
+```
+target_index_affinity true # defaults to false
+```
+
+By default plugin writes data of logstash format index based on current time. For example daily based index after mignight data is written to newly created index. This is normally ok when data is coming from single source and not updated after indexing.
+
+But if you have a use case where data is also updated after indexing and `id_key` is used to identify the document uniquely for updating. Logstash format is wanted to be used for easy data managing and retention. Updates are done right after indexing to complete the data (all data not available from single source) and no updates are done anymore later point on time. In this case problem happends at index rotation time where write to 2 indexes with same id_key value may happen.
+
+This setting will search existing data by using elastic search's [id query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-ids-query.html) using `id_key` value (with logstash_prefix and logstash_prefix_separator index pattarn e.g. `logstash-*`). The index of found data is used for update/upsert. When no data is found, data is written to current logstash index as normally.
+
+This setting requires following other settings:
+```
+logstash_format true
+id_key myId  # Some field on your data to identify the data uniquely
+write_operation upsert  # upsert or update
+```
+
+Suppose you have the following situation where you have 2 different match to consume data from 2 different Kafka topics independently but close in time with each other (order not known).
+
+```
+  <match data1>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+
+  <match data2>
+    @type elasticsearch
+    ...
+    id_key myId
+    write_operation upsert
+    logstash_format true
+    logstash_dateformat %Y.%m.%d
+    logstash_prefix myindexprefix
+    target_index_affinity true
+    ...
+```
+
+If your first (data1) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield1": "some value",
+}
+```
+
+and your second (data2) input is:
+```
+{
+  "myId": "myuniqueId1",
+  "datafield99": "some important data from other source tightly related to id myuniqueId1 and wanted to be in same document.",
+}
+```
+
+Date today is 10.05.2021 so data is written to index `myindexprefix-2021.05.10` when both data1 and data2 is consumed during today.
+But when we are close to index rotation and data1 is consumed and indexed at `2021-05-10T23:59:55.59707672Z` and data2
+is consumed a bit later at `2021-05-11T00:00:58.222079Z` i.e. logstash index has been rotated and normally data2 would have been written
+to index `myindexprefix-2021.05.11`. But with target_index_affinity setting as value true, data2 is now written to index `myindexprefix-2021.05.10`
+into same document with data1 as wanted and duplicated document is avoided.
+
 ### template_name
 
 The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
@@ -1451,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
 data_stream_name test
 ```
 
-When `@type elasticsearch_data_stream` is used, ILM default policy is set to the specified data stream.
+When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
 Then, the matching index template is also created automatically.
 
 ### data_stream_name
@@ -1459,6 +1529,18 @@ Then, the matching index template is also created automatically.
 You can specify Elasticsearch data stream name by this parameter.
 This parameter is mandatory for `elasticsearch_data_stream`.
 
+### data_stream_template_name
+
+You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template. 
+
+Default value is `data_stream_name`.
+
+### data_stream_ilm_name
+
+You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless `data_stream_template_name` is defined, in that case the ILM will be set to the one specified in the matching index template).
+
+Default value is `data_stream_name`.
+
 There are some limitations about naming rule.
 
 In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.3'
+  s.version       = '5.1.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_error?(type)
+    ['json_parse_exception'].include?(type)
+  end
+
   def log_es_400_reason(&block)
     if @plugin.log_es_400_reason
       block.call
@@ -43,15 +47,17 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     stats = Hash.new(0)
     meta = {}
     header = {}
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
     chunk.msgpack_each do |time, rawrecord|
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
         next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
+        @plugin.log.debug("Exception in error handler during deep copy: #{e}")
         stats[:bad_chunk_record] += 1
         next
       end
@@ -105,10 +111,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
           stats[type] += 1
-          retry_stream.add(time, rawrecord)
           if unrecoverable_error?(type)
             raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
           end
+          if unrecoverable_record_error?(type)
+            @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
+            next
+          else
+            retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
+          end
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -32,13 +32,25 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
+  def host_unreachable_exceptions
+    if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+      # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
+      # Elasticsearch class on Transport.
+      # This is why #host_unreachable_exceptions is not callable directly
+      # via transport (not transport's transport instance accessor) any more.
+      client.transport.transport.host_unreachable_exceptions
+    else
+      client.transport.host_unreachable_exceptions
+    end
+  end
+
   def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
     transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,6 +2,7 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+require 'set'
 begin
   require 'elasticsearch/xpack'
 rescue LoadError
@@ -71,7 +72,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
-    TARGET_BULK_BYTES = 20 * 1024 * 1024
+    DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
     config_param :host, :string,  :default => 'localhost'
@@ -165,7 +166,7 @@ EOC
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
-    config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
     config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
@@ -175,6 +176,7 @@ EOC
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
     config_param :catch_transport_exception_on_retry, :bool, :default => true
+    config_param :target_index_affinity, :bool, :default => false
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -834,13 +836,14 @@ EOC
                extract_placeholders(@host, chunk)
              end
 
+      affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
         record = inject_chunk_id_to_record_if_needed(record, chunk_id)
 
         begin
-          meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
+          meta, header, record = process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
           info = if @include_index_in_url
                    RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze], meta.delete("_alias".freeze))
                  else
@@ -877,6 +880,42 @@ EOC
       end
     end
 
+    def target_index_affinity_enabled?()
+      @target_index_affinity && @logstash_format && @id_key && (@write_operation == UPDATE_OP || @write_operation == UPSERT_OP)
+    end
+
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      if target_index_affinity_enabled?()
+        id_key_accessor = record_accessor_create(@id_key)
+        ids = Set.new
+        chunk.msgpack_each do |time, record|
+          next unless record.is_a? Hash
+          begin
+            ids << id_key_accessor.call(record)
+          end
+        end
+        log.debug("Find affinity target_indices by quering on ES (write_operation #{@write_operation}) for ids: #{ids.to_a}")
+        options = {
+          :index => "#{logstash_prefix}#{@logstash_prefix_separator}*",
+        }
+        query = {
+          'query' => { 'ids' => { 'values' => ids.to_a } },
+          '_source' => false,
+          'sort' => [
+            {"_index" => {"order" => "desc"}}
+         ]
+        }
+        result = client.search(options.merge(:body => Yajl.dump(query)))
+        # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
+        result['hits']['hits'].each do |hit|
+          indices[hit["_id"]] = hit["_index"]
+          log.debug("target_index for id: #{hit["_id"]} from es: #{hit["_index"]}")
+        end
+      end
+      indices
+    end
+
     def split_request?(bulk_message, info)
       # For safety.
     end
@@ -889,7 +928,7 @@ EOC
       false
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
@@ -930,6 +969,15 @@ EOC
         record[@tag_key] = tag
       end
 
+      # If affinity target indices map has value for this particular id, use it as target_index
+      if !affinity_target_indices.empty?
+        id_accessor = record_accessor_create(@id_key)
+        id_value = id_accessor.call(record)
+        if affinity_target_indices.key?(id_value)
+          target_index = affinity_target_indices[id_value]
+        end
+      end
+
       target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -1,3 +1,4 @@
+
 require_relative 'out_elasticsearch'
 
 module Fluent::Plugin
@@ -8,6 +9,8 @@ module Fluent::Plugin
     helpers :event_emitter
 
     config_param :data_stream_name, :string
+    config_param :data_stream_ilm_name, :string, :default => :data_stream_name
+    config_param :data_stream_template_name, :string, :default => :data_stream_name
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -26,7 +29,7 @@ module Fluent::Plugin
 
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
-        validate_data_stream_name
+        validate_data_stream_parameters
       else
         @use_placeholder = true
         @data_stream_names = []
@@ -36,8 +39,8 @@ module Fluent::Plugin
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name)
-          create_index_template(@data_stream_name)
+          create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
+          create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
           create_data_stream(@data_stream_name)
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -45,31 +48,35 @@ module Fluent::Plugin
       end
     end
 
-    def validate_data_stream_name
-      unless valid_data_stream_name?
-        unless start_with_valid_characters?
-          if not_dots?
-            raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
-          else
-            raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name,
+       "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
           end
-        end
-        unless valid_characters?
-          raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
-        end
-        unless lowercase_only?
-          raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
-        end
-        if @data_stream_name.bytes.size > 255
-          raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
         end
       end
     end
 
-    def create_ilm_policy(name)
-      return if data_stream_exist?(name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
       params = {
-        policy_id: "#{name}_policy",
+        policy_id: "#{ilm_name}_policy",
         body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
       }
       retry_operate(@max_retry_putting_template,
@@ -79,19 +86,19 @@ module Fluent::Plugin
       end
     end
 
-    def create_index_template(name)
-      return if data_stream_exist?(name)
+    def create_index_template(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
       body = {
-        "index_patterns" => ["#{name}*"],
+        "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
         "template" => {
           "settings" => {
-            "index.lifecycle.name" => "#{name}_policy"
+            "index.lifecycle.name" => "#{ilm_name}_policy"
           }
         }
       }
       params = {
-        name: name,
+        name: template_name,
         body: body
       }
       retry_operate(@max_retry_putting_template,
@@ -101,9 +108,9 @@ module Fluent::Plugin
       end
     end
 
-    def data_stream_exist?(name)
+    def data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       begin
         response = @client.indices.get_data_stream(params)
@@ -114,10 +121,10 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(name)
-      return if data_stream_exist?(name)
+    def create_data_stream(datastream_name)
+      return if data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
@@ -126,28 +133,48 @@ module Fluent::Plugin
       end
     end
 
-    def valid_data_stream_name?
-      lowercase_only? and
-        valid_characters? and
-        start_with_valid_characters? and
-        not_dots? and
-        @data_stream_name.bytes.size <= 255
+    def ilm_policy_exists?(policy_id)
+      begin
+        @client.ilm.get_policy(policy_id: policy_id)
+        true
+      rescue
+        false
+      end
+    end
+
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
     end
 
-    def lowercase_only?
-      @data_stream_name.downcase == @data_stream_name
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
     end
 
-    def valid_characters?
-      not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
     end
 
-    def start_with_valid_characters?
-      not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
     end
 
-    def not_dots?
-      not (@data_stream_name == "." or @data_stream_name == "..")
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
     end
 
     def client_library_version
@@ -160,13 +187,18 @@ module Fluent::Plugin
 
     def write(chunk)
       data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      data_stream_ilm_name = @data_stream_ilm_name
+      host = @host
       if @use_placeholder
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
         unless @data_stream_names.include?(data_stream_name)
           begin
-            create_ilm_policy(data_stream_name)
-            create_index_template(data_stream_name)
             create_data_stream(data_stream_name)
+            create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
             @data_stream_names << data_stream_name
           rescue => e
             raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
@@ -200,7 +232,7 @@ module Fluent::Plugin
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end
       rescue => e
-        log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
       end
     end