fluent-plugin-elasticsearch 5.1.0 → 5.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89dfb07388fcdb941bd3dc13a9e481fcb2ecf8f4a0b3c828b6a539f13c90e8ee
-  data.tar.gz: ee8ac22c05d144076e62311e013fea483d72493d6066f87712b5c325f3c215e7
+  metadata.gz: 9cbe785cfd5534abf0b55d8207eae176d1dd02d5ef7a3092e9f0ef3c6d588e7f
+  data.tar.gz: c12b413d1415ac38a6d119ff1f0ca106b1d4f1017f206ce72d97a74abe819551
 SHA512:
-  metadata.gz: a69fbfa9c0fab591f55656d87cc99aa003aa4e3b7a7d1b2819bd385e70d0c67d7b91dfff0fa42cfeafe94b6fe8f467b069ed075a2776809d0615f6e243f8c355
-  data.tar.gz: 572585eeb9e939d2b08e5fe0590bd25f02b8390a730d9f3012c44604b33123d9372cf59ac50a108119b413f3d718e2a606a0b21dc62f676199329e3b12c6073c
+  metadata.gz: d30472143db2451da2eca168b22086d157abb7e62c26f1df5afa2ab4ed23a250a2b81fd04424f260dbfe996cc0a4794504e0c4262efc7969db363be02c79bcf9
+  data.tar.gz: 9fd7a57fe732f18b6f568a96052ba7a5bf51423e93072ba230c7a57866b828dfe07e16b11f937c5d84d0cb01619fe027ccd80c0a0a4fb939d94a41f127ff5e3c

data/History.md

@@ -2,6 +2,21 @@
 
 ### [Unreleased]
 
+### 5.1.4
+- Handle ES8 or above more strictly (#931)
+- fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)
+
+### 5.1.3
+-  fixing execution order for dynamic data stream creation (#928)
+
+### 5.1.2
+-  Fix default values of datastream parameters (#926)
+
+### 5.1.1
+- Report appropriate error for data_stream parameters (#922)
+- Add ILM and template parameters for data streams (#920)
+- Support Buffer in Data Stream Output (#917)
+
 ### 5.1.0
 - Correct default target bytes value (#914)
 - Handle elasticsearch-ruby 7.14 properly (#913)

data/README.md

@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
 data_stream_name test
 ```
 
-When `@type elasticsearch_data_stream` is used, ILM default policy is set to the specified data stream.
+When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
 Then, the matching index template is also created automatically.
 
 ### data_stream_name
@@ -1529,6 +1529,18 @@ Then, the matching index template is also created automatically.
 You can specify Elasticsearch data stream name by this parameter.
 This parameter is mandatory for `elasticsearch_data_stream`.
 
+### data_stream_template_name
+
+You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template. 
+
+Default value is `data_stream_name`.
+
+### data_stream_ilm_name
+
+You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless `data_stream_template_name` is defined, in that case the ILM will be set to the one specified in the matching index template).
+
+Default value is `data_stream_name`.
+
 There are some limitations about naming rule.
 
 In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.1.0'
+  s.version       = '5.1.4'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -987,12 +987,12 @@ EOC
         elsif @last_seen_major_version == 7
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
-        elsif @last_seen_major_version >=8
+        elsif @last_seen_major_version >= 8
           log.debug "Detected ES 8.x or above: document type will not be used."
           target_type = nil
         end
       else
-        if @suppress_type_name && @last_seen_major_version >= 7
+        if @suppress_type_name && @last_seen_major_version == 7
           target_type = nil
         elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -1,3 +1,4 @@
+
 require_relative 'out_elasticsearch'
 
 module Fluent::Plugin
@@ -8,6 +9,8 @@ module Fluent::Plugin
     helpers :event_emitter
 
     config_param :data_stream_name, :string
+    config_param :data_stream_ilm_name, :string, :default => nil
+    config_param :data_stream_template_name, :string, :default => nil
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -24,9 +27,12 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
       end
 
+      @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
-        validate_data_stream_name
+        validate_data_stream_parameters
       else
         @use_placeholder = true
         @data_stream_names = []
@@ -36,8 +42,8 @@ module Fluent::Plugin
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name)
-          create_index_template(@data_stream_name)
+          create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
+          create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
           create_data_stream(@data_stream_name)
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -45,31 +51,35 @@ module Fluent::Plugin
       end
     end
 
-    def validate_data_stream_name
-      unless valid_data_stream_name?
-        unless start_with_valid_characters?
-          if not_dots?
-            raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
-          else
-            raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name,
+       "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
           end
-        end
-        unless valid_characters?
-          raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
-        end
-        unless lowercase_only?
-          raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
-        end
-        if @data_stream_name.bytes.size > 255
-          raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
         end
       end
     end
 
-    def create_ilm_policy(name)
-      return if data_stream_exist?(name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
       params = {
-        policy_id: "#{name}_policy",
+        policy_id: ilm_name,
         body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
       }
       retry_operate(@max_retry_putting_template,
@@ -79,19 +89,19 @@ module Fluent::Plugin
       end
     end
 
-    def create_index_template(name)
-      return if data_stream_exist?(name)
+    def create_index_template(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
       body = {
-        "index_patterns" => ["#{name}*"],
+        "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
         "template" => {
           "settings" => {
-            "index.lifecycle.name" => "#{name}_policy"
+            "index.lifecycle.name" => "#{ilm_name}"
           }
         }
       }
       params = {
-        name: name,
+        name: template_name,
         body: body
       }
       retry_operate(@max_retry_putting_template,
@@ -101,9 +111,9 @@ module Fluent::Plugin
       end
     end
 
-    def data_stream_exist?(name)
+    def data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       begin
         response = @client.indices.get_data_stream(params)
@@ -114,10 +124,10 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(name)
-      return if data_stream_exist?(name)
+    def create_data_stream(datastream_name)
+      return if data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
@@ -126,28 +136,48 @@ module Fluent::Plugin
       end
     end
 
-    def valid_data_stream_name?
-      lowercase_only? and
-        valid_characters? and
-        start_with_valid_characters? and
-        not_dots? and
-        @data_stream_name.bytes.size <= 255
+    def ilm_policy_exists?(policy_id)
+      begin
+        @client.ilm.get_policy(policy_id: policy_id)
+        true
+      rescue
+        false
+      end
+    end
+
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
     end
 
-    def lowercase_only?
-      @data_stream_name.downcase == @data_stream_name
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
     end
 
-    def valid_characters?
-      not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
     end
 
-    def start_with_valid_characters?
-      not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
     end
 
-    def not_dots?
-      not (@data_stream_name == "." or @data_stream_name == "..")
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
     end
 
     def client_library_version
@@ -160,12 +190,17 @@ module Fluent::Plugin
 
     def write(chunk)
       data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      data_stream_ilm_name = @data_stream_ilm_name
+      host = @host
       if @use_placeholder
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
         unless @data_stream_names.include?(data_stream_name)
           begin
-            create_ilm_policy(data_stream_name)
-            create_index_template(data_stream_name)
+            create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
             create_data_stream(data_stream_name)
             @data_stream_names << data_stream_name
           rescue => e
@@ -200,7 +235,7 @@ module Fluent::Plugin
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end
       rescue => e
-        log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
       end
     end
 

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -45,7 +45,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     {
       'data_streams': [
                         {
-                          'name' => 'my-data-stream',
+                          'name' => 'foo',
                           'timestamp_field' => {
                             'name' => '@timestamp'
                           }
@@ -62,11 +62,11 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
 
-  def stub_ilm_policy(name="foo")
-    stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}_policy").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_ilm_policy(name="foo_ilm_policy")
+    stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_index_template(name="foo")
+  def stub_index_template(name="foo_tpl")
     stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
@@ -78,14 +78,42 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
+  def stub_existent_ilm?(name="foo_ilm_policy")
+    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  end
+
+  def stub_existent_template?(name="foo_tpl")
+    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  end
+
   def stub_nonexistent_data_stream?(name="foo")
     stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_bulk_feed(name="foo")
-    stub_request(:post, "http://localhost:9200/#{name}/_bulk").with do |req|
+  def stub_nonexistent_ilm?(name="foo_ilm_policy")
+    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  end
+
+  def stub_nonexistent_template?(name="foo_tpl")
+    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  end
+
+  def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl")
+    stub_request(:post, "http://localhost:9200/#{datastream_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
-      # {"create": {}}\n
+      # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
+      # {"@timestamp": ...}
+      @bulk_records += req.body.split("\n").size / 2
+    end
+    stub_request(:post, "http://localhost:9200/#{ilm_name}/_bulk").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
+      # {"@timestamp": ...}
+      @bulk_records += req.body.split("\n").size / 2
+    end
+    stub_request(:post, "http://localhost:9200/#{template_name}/_bulk").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
@@ -96,12 +124,14 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
   end
 
-  def stub_default(name="foo", host="http://localhost:9200")
+  def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
     stub_elastic_info(host)
-    stub_ilm_policy(name)
-    stub_index_template(name)
-    stub_nonexistent_data_stream?(name)
-    stub_data_stream(name)
+    stub_nonexistent_ilm?(ilm_name)
+    stub_ilm_policy(ilm_name)
+    stub_nonexistent_template?(template_name)
+    stub_index_template(template_name)
+    stub_nonexistent_data_stream?(datastream_name)
+    stub_data_stream(datastream_name)
   end
 
   def data_stream_supported?
@@ -121,82 +151,274 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       end
     end
 
-    def test_invalid_uppercase
-      conf = config_element(
-        'ROOT', '', {
-          '@type' => 'elasticsearch_datastream',
-          'data_stream_name' => 'TEST'
-        })
-      assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
-        driver(conf)
+    sub_test_case "invalid uppercase" do
+      def test_stream_name
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => 'elasticsearch_datastream',
+            'data_stream_name' => 'TEST',
+            'data_stream_ilm_name' => 'default-policy',
+            'data_stream_template_name' => 'template'
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
+          driver(conf)
+        end
+      end
+      def test_stream_ilm_name
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => 'elasticsearch_datastream',
+            'data_stream_name' => 'data_stream',
+            'data_stream_ilm_name' => 'TEST-ILM',
+            'data_stream_template_name' => 'template'
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must be lowercase only: <TEST-ILM>") do
+          driver(conf)
+        end
+      end
+      def test_stream_template_name
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => 'elasticsearch_datastream',
+            'data_stream_name' => 'default',
+            'data_stream_ilm_name' => 'default-policy',
+            'data_stream_template_name' => 'TEST-TPL'
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_template_name' must be lowercase only: <TEST-TPL>") do
+          driver(conf)
+        end
       end
     end
 
-    data("backslash" => "\\",
-         "slash" => "/",
-         "asterisk" => "*",
-         "question" => "?",
-         "doublequote" => "\"",
-         "lt" => "<",
-         "gt" => ">",
-         "bar" => "|",
-         "space" => " ",
-         "comma" => ",",
-         "sharp" => "#",
-         "colon" => ":")
-    def test_invalid_characters(data)
-      c, _ = data
-      conf = config_element(
-        'ROOT', '', {
-          '@type' => ELASTIC_DATA_STREAM_TYPE,
-          'data_stream_name' => "TEST#{c}"
-        })
-      label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
-      assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
-        driver(conf)
+    sub_test_case "invalid parameters" do
+      data("backslash" => "\\",
+           "slash" => "/",
+           "asterisk" => "*",
+           "question" => "?",
+           "doublequote" => "\"",
+           "lt" => "<",
+           "gt" => ">",
+           "bar" => "|",
+           "space" => " ",
+           "comma" => ",",
+           "sharp" => "#",
+           "colon" => ":")
+      def test_stream_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "TEST#{c}",
+            'data_stream_ilm_name' => "default_policy",
+            'data_stream_template_name' => "data_stream"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
+          driver(conf)
+        end
+      end
+
+      data("backslash" => "\\",
+           "slash" => "/",
+           "asterisk" => "*",
+           "question" => "?",
+           "doublequote" => "\"",
+           "lt" => "<",
+           "gt" => ">",
+           "bar" => "|",
+           "space" => " ",
+           "comma" => ",",
+           "sharp" => "#",
+           "colon" => ":")
+      def test_stream_ilm_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "TEST#{c}",
+            'data_stream_template_name' => "data_stream"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not contain invalid characters #{label}: <TEST#{c}>") do
+          driver(conf)
+        end
+      end
+
+      data("backslash" => "\\",
+           "slash" => "/",
+           "asterisk" => "*",
+           "question" => "?",
+           "doublequote" => "\"",
+           "lt" => "<",
+           "gt" => ">",
+           "bar" => "|",
+           "space" => " ",
+           "comma" => ",",
+           "sharp" => "#",
+           "colon" => ":")
+      def test_stream_template_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "default_policy",
+            'data_stream_template_name' => "TEST#{c}"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not contain invalid characters #{label}: <TEST#{c}>") do
+          driver(conf)
+        end
       end
     end
 
-    data("hyphen" => "-",
-         "underscore" => "_",
-         "plus" => "+",
-         "period" => ".")
-    def test_invalid_start_characters(data)
-      c, _ = data
-      conf = config_element(
-        'ROOT', '', {
-          '@type' => ELASTIC_DATA_STREAM_TYPE,
-          'data_stream_name' => "#{c}TEST"
-        })
-      label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
-      assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
-        driver(conf)
+    sub_test_case "invalid start characters" do
+      data("hyphen" => "-",
+           "underscore" => "_",
+           "plus" => "+",
+           "period" => ".")
+      def test_stream_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "#{c}TEST",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "template"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
+          driver(conf)
+        end
+      end
+      data("hyphen" => "-",
+           "underscore" => "_",
+           "plus" => "+",
+           "period" => ".")
+      def test_stream_ilm_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "#{c}TEST",
+            'data_stream_template_name' => "template"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not start with #{label}: <#{c}TEST>") do
+          driver(conf)
+        end
+      end
+      data("hyphen" => "-",
+           "underscore" => "_",
+           "plus" => "+",
+           "period" => ".")
+      def test_stream_template_name(data)
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "#{c}TEST"
+          })
+        label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
+        assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not start with #{label}: <#{c}TEST>") do
+          driver(conf)
+        end
       end
     end
 
-    data("current" => ".",
-         "parents" => "..")
-    def test_invalid_dots
-      c, _ = data
-      conf = config_element(
-        'ROOT', '', {
-          '@type' => ELASTIC_DATA_STREAM_TYPE,
-          'data_stream_name' => "#{c}"
-        })
-      assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
-        driver(conf)
+    sub_test_case "invalid dots" do
+      data("current" => ".",
+           "parents" => "..")
+      def test_stream_name
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "#{c}",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "template"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
+          driver(conf)
+        end
+      end
+
+      data("current" => ".",
+           "parents" => "..")
+      def test_stream_ilm_name
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "#{c}",
+            'data_stream_template_name' => "template"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be . or ..: <#{c}>") do
+          driver(conf)
+        end
+      end
+
+      data("current" => ".",
+           "parents" => "..")
+      def test_stream_template_name
+        c, _ = data
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "#{c}"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be . or ..: <#{c}>") do
+          driver(conf)
+        end
       end
     end
 
-    def test_invalid_length
-      c = "a" * 256
-      conf = config_element(
-        'ROOT', '', {
-          '@type' => ELASTIC_DATA_STREAM_TYPE,
-          'data_stream_name' => "#{c}"
-        })
-      assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
-        driver(conf)
+    sub_test_case "invalid length" do
+      def test_stream_name
+        c = "a" * 256
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "#{c}",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "template"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
+          driver(conf)
+        end
+      end
+      def test_stream_ilm_name
+        c = "a" * 256
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "#{c}",
+            'data_stream_template_name' => "template"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be longer than 255 bytes: <#{c}>") do
+          driver(conf)
+        end
+      end
+      def test_stream_template_name
+        c = "a" * 256
+        conf = config_element(
+          'ROOT', '', {
+            '@type' => ELASTIC_DATA_STREAM_TYPE,
+            'data_stream_name' => "default",
+            'data_stream_ilm_name' => "default-policy",
+            'data_stream_template_name' => "#{c}"
+          })
+        assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be longer than 255 bytes: <#{c}>") do
+          driver(conf)
+        end
       end
     end
   end
@@ -208,7 +430,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo'
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => "foo_ilm_policy",
+        'data_stream_template_name' => "foo_tpl"
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
@@ -224,21 +448,101 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo'
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => "foo_ilm_policy",
+        'data_stream_template_name' => "foo_tpl"
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
+  def test_template_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => "foo_ilm_policy",
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
+    assert_equal "foo_template", driver(conf).instance.data_stream_template_name
+  end
+
+  def test_ilm_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_template_name' => "foo_tpl"
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_tpl", driver(conf).instance.data_stream_template_name
+  end
+
+  def test_template_and_ilm_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_template", driver(conf).instance.data_stream_template_name
+    assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
+  end
+
   def test_placeholder
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
-    name = "foo_test"
-    stub_default(name)
-    stub_bulk_feed(name)
+    dsname = "foo_test"
+    ilmname = "foo_ilm_test"
+    tplname = "foo_tpl_test"
+    stub_default(dsname, ilmname, tplname)
+    stub_bulk_feed(dsname, ilmname, tplname)
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo_${tag}',
+        'data_stream_ilm_name' => "foo_ilm_${tag}",
+        'data_stream_template_name' => "foo_tpl_${tag}"
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records
+  end
+
+  def test_placeholder_params_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    dsname = "foo_test"
+    ilmname = "foo_test_policy"
+    tplname = "foo_test_template"
+    stub_default(dsname, ilmname, tplname)
+    stub_bulk_feed(dsname, ilmname, tplname)
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo_${tag}'
+        'data_stream_name' => 'foo_${tag}',
       })
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -246,17 +550,22 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records
   end
 
+
   def test_time_placeholder
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
     time = Time.now
-    name = "foo_#{time.strftime("%Y%m%d")}"
-    stub_default(name)
-    stub_bulk_feed(name)
+    dsname = "foo_#{time.strftime("%Y%m%d")}"
+    ilmname = "foo_ilm_#{time.strftime("%Y%m%d")}"
+    tplname = "foo_tpl_#{time.strftime("%Y%m%d")}"
+    stub_default(dsname, ilmname, tplname)
+    stub_bulk_feed(dsname, ilmname, tplname)
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo_%Y%m%d'
+        'data_stream_name' => 'foo_%Y%m%d',
+        'data_stream_ilm_name' => 'foo_ilm_%Y%m%d',
+        'data_stream_template_name' => 'foo_tpl_%Y%m%d'
       }, [config_element('buffer', 'time', {
                           'timekey' => '1d'
                         }, [])]
@@ -272,14 +581,18 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
 
     keys = ["bar", "baz"]
     keys.each do |key|
-      name = "foo_#{key}"
-      stub_default(name)
-      stub_bulk_feed(name)
+      dsname = "foo_#{key}"
+      ilmname = "foo_ilm_#{key}"
+      tplname = "foo_tpl_#{key}"
+      stub_default(dsname, ilmname, tplname)
+      stub_bulk_feed(dsname, ilmname, tplname)
     end
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo_${key1}'
+        'data_stream_name' => 'foo_${key1}',
+        'data_stream_ilm_name' => 'foo_ilm_${key1}',
+        'data_stream_template_name' => 'foo_tpl_${key1}'
       }, [config_element('buffer', 'tag,key1', {
                           'timekey' => '1d'
                         }, [])]
@@ -301,7 +614,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     conf = config_element(
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
-        'data_stream_name' => 'foo'
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl'
       })
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -316,14 +631,16 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     template_file = File.join(cwd, 'test_index_template.json')
 
     config = %{
-      host            logs.google.com
-      port            778
-      scheme          https
-      data_stream_name foo
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
+      host                       logs.google.com
+      port                       778
+      scheme                     https
+      data_stream_name           foo
+      data_stream_ilm_name       foo_ilm_policy
+      data_stream_template_name  foo_tpl
+      user                       john
+      password                   doe
+      template_name              logstash
+      template_file              #{template_file}
       max_retry_putting_template 3
     }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.1.0
+  version: 5.1.4
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-19 00:00:00.000000000 Z
+date: 2021-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -156,7 +156,6 @@ files:
 - ".github/workflows/macos.yml"
 - ".github/workflows/windows.yml"
 - ".gitignore"
-- ".travis.yml"
 - CONTRIBUTING.md
 - Gemfile
 - History.md
@@ -168,7 +167,6 @@ files:
 - README.Troubleshooting.md
 - README.md
 - Rakefile
-- appveyor.yml
 - fluent-plugin-elasticsearch.gemspec
 - gemfiles/Gemfile.elasticsearch.v6
 - lib/fluent/log-ext.rb
@@ -223,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.30
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector

data/.travis.yml

@@ -1,40 +0,0 @@
-language: ruby
-
-jobs:
-  include:
-    - rvm: 2.4.6
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.5.5
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: gemfiles/Gemfile.elasticsearch.v6
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: linux
-      arch: arm64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: osx
-      osx_image: xcode11.3
-    - rvm: 2.7.0
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-
-gemfile:
- - Gemfile
-
-before_install:
-  - gem update --system=2.7.8
-
-script: bundle exec rake test

data/appveyor.yml

@@ -1,20 +0,0 @@
-version: '{build}'
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - ridk.cmd enable
-  - ruby --version
-  - gem --version
-  - bundle install
-build: off
-test_script:
-  - bundle exec rake test
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "26-x64"
-    - ruby_version: "26"
-    - ruby_version: "25-x64"
-    - ruby_version: "25"
-    - ruby_version: "24-x64"
-    - ruby_version: "24"