fluent-plugin-elasticsearch 5.1.1 → 5.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27d74e7048671def02b98e337c052c395152021a4a3f4c2138d1780c725d09bd
-  data.tar.gz: eb5282b8e688b091700a549c711af2f1959bc9b1b2c4f6fd1b49e3119c62ddb7
+  metadata.gz: 0ae6214f54783c402097e26276b9247f1d58866b08cdc80be47d3b4638516d54
+  data.tar.gz: b69af374d04f84b2b1137189916c2b78f25eae71c2294d9a0cc196c30c09db29
 SHA512:
-  metadata.gz: 3a3ad9fa5259fcd1e80a85bdf7d1acd11cd26675d4f47f326100db242f0f9320232530099eb5346e5ea11aba76c4cc66cfc2e97f6393ca95d1227281217283ea
-  data.tar.gz: f97182a9487be71d34ddcd8ec2dd046eca9c6de1cae55d3842feeeaef627f23a05862be4783aee37fe38c84cba3bf984a51fb7fb3e8bad50f1bf0f57c956803e
+  metadata.gz: 29173c2331bc0e533aa1d3aa66aa555bad7802e5fbb4207641b11aebc0a18c0d4e44ebed0b167c8fe4bddf403d2dd5a61349afeffecfc28a65359c3bb4cce26b
+  data.tar.gz: 7fdf58624182bdd4e354cad2d4f9ba001cff5a8c8c0ba02ac043607a87e2372311deb85bc0306d3aa167e98170a6049f33f3203b202ebcf25eac75e65dc23a17

data/History.md

@@ -2,8 +2,23 @@
 
 ### [Unreleased]
 
+### 5.1.5
+- Make retryable DataStreams creation at configure phase (#943)
+- Handle @hosts parameter on data_stream plugin (#942)
+- allow specifying custom ILM policies for data streams (#933)
+
+### 5.1.4
+- Handle ES8 or above more strictly (#931)
+- fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)
+
+### 5.1.3
+-  fixing execution order for dynamic data stream creation (#928)
+
+### 5.1.2
+-  Fix default values of datastream parameters (#926)
+
 ### 5.1.1
--  Report appropriate error for data_stream parameters (#922)
+- Report appropriate error for data_stream parameters (#922)
 - Add ILM and template parameters for data streams (#920)
 - Support Buffer in Data Stream Output (#917)
 

data/README.md

@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
 data_stream_name test
 ```
 
-When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
+When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name` or `data_stream_ilm_policy`, ILM default policy is set to the specified data stream.
 Then, the matching index template is also created automatically.
 
 ### data_stream_name
@@ -1545,6 +1545,21 @@ There are some limitations about naming rule.
 
 In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
 
+
+### data_stream_ilm_policy
+
+You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
+### data_stream_ilm_policy_overwrite
+
+Specify whether the data stream ILM policy should be overwritten.
+
+Default value is `false`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.1.1'
+  s.version       = '5.1.5'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -987,12 +987,12 @@ EOC
         elsif @last_seen_major_version == 7
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
-        elsif @last_seen_major_version >=8
+        elsif @last_seen_major_version >= 8
           log.debug "Detected ES 8.x or above: document type will not be used."
           target_type = nil
         end
       else
-        if @suppress_type_name && @last_seen_major_version >= 7
+        if @suppress_type_name && @last_seen_major_version == 7
           target_type = nil
         elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -9,8 +9,11 @@ module Fluent::Plugin
     helpers :event_emitter
 
     config_param :data_stream_name, :string
-    config_param :data_stream_ilm_name, :string, :default => :data_stream_name
-    config_param :data_stream_template_name, :string, :default => :data_stream_name
+    config_param :data_stream_ilm_name, :string, :default => nil
+    config_param :data_stream_template_name, :string, :default => nil
+    config_param :data_stream_ilm_policy, :string, :default => nil
+    config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -27,6 +30,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
       end
 
+      @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+      @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
+
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
         validate_data_stream_parameters
@@ -35,13 +42,16 @@ module Fluent::Plugin
         @data_stream_names = []
       end
 
-      @client = client
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
-          create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
-          create_data_stream(@data_stream_name)
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_data_stream(@data_stream_name)
+          end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
         end
@@ -73,27 +83,30 @@ module Fluent::Plugin
       end
     end
 
-    def create_ilm_policy(datastream_name, template_name, ilm_name, host)
-      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
+      unless @data_stream_ilm_policy_overwrite
+        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
+      end
+
       params = {
-        policy_id: "#{ilm_name}_policy",
-        body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
+        policy_id: ilm_name,
+        body: @data_stream_ilm_policy
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.xpack.ilm.put_policy(params)
+        client(host).xpack.ilm.put_policy(params)
       end
     end
 
-    def create_index_template(datastream_name, template_name, ilm_name, host)
-      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
+    def create_index_template(datastream_name, template_name, ilm_name, host = nil)
+      return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
       body = {
         "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
         "template" => {
           "settings" => {
-            "index.lifecycle.name" => "#{ilm_name}_policy"
+            "index.lifecycle.name" => "#{ilm_name}"
           }
         }
       }
@@ -104,16 +117,16 @@ module Fluent::Plugin
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.put_index_template(params)
+        client(host).indices.put_index_template(params)
       end
     end
 
-    def data_stream_exist?(datastream_name)
+    def data_stream_exist?(datastream_name, host = nil)
       params = {
         name: datastream_name
       }
       begin
-        response = @client.indices.get_data_stream(params)
+        response = client(host).indices.get_data_stream(params)
         return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
       rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
         log.info "Specified data stream does not exist. Will be created: <#{e}>"
@@ -121,21 +134,21 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(datastream_name)
-      return if data_stream_exist?(datastream_name)
+    def create_data_stream(datastream_name, host = nil)
+      return if data_stream_exist?(datastream_name, host)
       params = {
         name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.create_data_stream(params)
+        client(host).indices.create_data_stream(params)
       end
     end
 
-    def ilm_policy_exists?(policy_id)
+    def ilm_policy_exists?(policy_id, host = nil)
       begin
-        @client.ilm.get_policy(policy_id: policy_id)
+        client(host).ilm.get_policy(policy_id: policy_id)
         true
       rescue
         false
@@ -189,16 +202,17 @@ module Fluent::Plugin
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
       data_stream_ilm_name = @data_stream_ilm_name
-      host = @host
+      host = nil
       if @use_placeholder
+        host = extract_placeholders(@host, chunk)
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
         data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
         unless @data_stream_names.include?(data_stream_name)
           begin
-            create_data_stream(data_stream_name)
             create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
             create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_data_stream(data_stream_name)
             @data_stream_names << data_stream_name
           rescue => e
             raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
@@ -227,7 +241,7 @@ module Fluent::Plugin
         body: bulk_message
       }
       begin
-        response = @client.bulk(params)
+        response = client(host).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -62,56 +62,61 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
 
-  def stub_ilm_policy(name="foo_ilm")
-    stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}_policy").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_index_template(name="foo_tpl")
-    stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_index_template(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_data_stream(name="foo")
-    stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_data_stream(name="foo", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_ilm?(name="foo_ilm")
-    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_nonexistent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_nonexistent_ilm?(name="foo_ilm")
-    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_nonexistent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm", template_name="foo_tpl")
-    stub_request(:post, "http://localhost:9200/#{datastream_name}/_bulk").with do |req|
+  def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").
+      to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
+  end
+
+  def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
+    stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
-    stub_request(:post, "http://localhost:9200/#{ilm_name}/_bulk").with do |req|
+    stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
-    stub_request(:post, "http://localhost:9200/#{template_name}/_bulk").with do |req|
+    stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
@@ -119,12 +124,12 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0", headers={})
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
   end
 
-  def stub_default(datastream_name="foo", ilm_name="foo_ilm", template_name="foo_tpl", host="http://localhost:9200")
+  def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
     stub_elastic_info(host)
     stub_nonexistent_ilm?(ilm_name)
     stub_ilm_policy(ilm_name)
@@ -431,12 +436,62 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
-        'data_stream_ilm_name' => "foo_ilm",
+        'data_stream_ilm_name' => "foo_ilm_policy",
         'data_stream_template_name' => "foo_tpl"
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
+  def test_datastream_configure_retry
+    stub_elastic_info
+    stub_nonexistent_ilm?
+    stub_ilm_policy
+    stub_nonexistent_template_retry?
+    stub_index_template
+    stub_nonexistent_data_stream?
+    stub_data_stream
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => "foo_ilm_policy",
+      'data_stream_template_name' => "foo_tpl"
+    })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+  end
+
+  def test_hosts_list_configure
+    config = %{
+      hosts            https://john:password@host1:443/elastic/,http://host2
+      path             /default_path
+      user             default_user
+      password         default_password
+      data_stream_name default
+    }
+    stub_elastic_info("https://host1:443/elastic//", "7.9.0",
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
+    stub_elastic_info("http://host2/default_path/_data_stream/default", "7.9.0",
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
+    stub_existent_data_stream?("default", "https://host1/elastic/")
+    instance = driver(config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'password', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_existent_data_stream
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -449,10 +504,65 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
-        'data_stream_ilm_name' => "foo_ilm",
+        'data_stream_ilm_name' => "foo_ilm_policy",
+        'data_stream_template_name' => "foo_tpl"
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+  end
+
+  def test_template_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => "foo_ilm_policy",
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
+    assert_equal "foo_template", driver(conf).instance.data_stream_template_name
+  end
+
+  def test_ilm_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
         'data_stream_template_name' => "foo_tpl"
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_tpl", driver(conf).instance.data_stream_template_name
+  end
+
+  def test_template_and_ilm_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_ilm_policy
+    stub_index_template
+    stub_existent_data_stream?
+    stub_data_stream
+    stub_elastic_info
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+      })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+    assert_equal "foo_template", driver(conf).instance.data_stream_template_name
+    assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
   end
 
   def test_placeholder
@@ -476,6 +586,26 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records
   end
 
+  def test_placeholder_params_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    dsname = "foo_test"
+    ilmname = "foo_test_policy"
+    tplname = "foo_test_template"
+    stub_default(dsname, ilmname, tplname)
+    stub_bulk_feed(dsname, ilmname, tplname)
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo_${tag}',
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records
+  end
+
+
   def test_time_placeholder
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -540,7 +670,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       'ROOT', '', {
         '@type' => ELASTIC_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
-        'data_stream_ilm_name' => 'foo_ilm',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
         'data_stream_template_name' => 'foo_tpl'
       })
     driver(conf).run(default_tag: 'test') do
@@ -560,7 +690,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       port                       778
       scheme                     https
       data_stream_name           foo
-      data_stream_ilm_name       foo_ilm
+      data_stream_ilm_name       foo_ilm_policy
       data_stream_template_name  foo_tpl
       user                       john
       password                   doe
@@ -584,4 +714,86 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
 
     assert_equal(4, connection_resets)
   end
+
+  def test_doesnt_update_ilm_policy_if_overwrite_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+    }
+
+    stub_elastic_info
+    stub_index_template
+    stub_existent_data_stream?
+    stub_existent_ilm?
+    stub_data_stream
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 0)
+  end
+
+  def test_updates_ilm_policy_if_overwrite_set
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+      data_stream_ilm_policy_overwrite  true
+    }
+
+    stub_elastic_info
+    stub_index_template
+    stub_existent_data_stream?
+    stub_existent_ilm?
+    stub_data_stream
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+      body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
+      times: 1)
+  end
+
+  def test_creates_custom_ilm_policy_if_none_exists
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+    }
+
+    stub_elastic_info
+    stub_index_template("foo_template")
+    stub_data_stream
+    stub_nonexistent_data_stream?
+    stub_nonexistent_ilm?
+    stub_nonexistent_template?("foo_template")
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+      body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
+      times: 1)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.1.1
+  version: 5.1.5
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -156,7 +156,6 @@ files:
 - ".github/workflows/macos.yml"
 - ".github/workflows/windows.yml"
 - ".gitignore"
-- ".travis.yml"
 - CONTRIBUTING.md
 - Gemfile
 - History.md
@@ -168,7 +167,6 @@ files:
 - README.Troubleshooting.md
 - README.md
 - Rakefile
-- appveyor.yml
 - fluent-plugin-elasticsearch.gemspec
 - gemfiles/Gemfile.elasticsearch.v6
 - lib/fluent/log-ext.rb
@@ -223,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.2.30
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector

data/.travis.yml

@@ -1,40 +0,0 @@
-language: ruby
-
-jobs:
-  include:
-    - rvm: 2.4.6
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.5.5
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: gemfiles/Gemfile.elasticsearch.v6
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: linux
-      arch: arm64
-    - rvm: 2.6.3
-      gemfile: Gemfile
-      os: osx
-      osx_image: xcode11.3
-    - rvm: 2.7.0
-      gemfile: Gemfile
-      os: linux
-      arch: amd64
-
-gemfile:
- - Gemfile
-
-before_install:
-  - gem update --system=2.7.8
-
-script: bundle exec rake test

data/appveyor.yml

@@ -1,20 +0,0 @@
-version: '{build}'
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - ridk.cmd enable
-  - ruby --version
-  - gem --version
-  - bundle install
-build: off
-test_script:
-  - bundle exec rake test
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "26-x64"
-    - ruby_version: "26"
-    - ruby_version: "25-x64"
-    - ruby_version: "25"
-    - ruby_version: "24-x64"
-    - ruby_version: "24"