fluent-plugin-elasticsearch 5.1.4 → 5.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9cbe785cfd5534abf0b55d8207eae176d1dd02d5ef7a3092e9f0ef3c6d588e7f
-  data.tar.gz: c12b413d1415ac38a6d119ff1f0ca106b1d4f1017f206ce72d97a74abe819551
+  metadata.gz: 0ae6214f54783c402097e26276b9247f1d58866b08cdc80be47d3b4638516d54
+  data.tar.gz: b69af374d04f84b2b1137189916c2b78f25eae71c2294d9a0cc196c30c09db29
 SHA512:
-  metadata.gz: d30472143db2451da2eca168b22086d157abb7e62c26f1df5afa2ab4ed23a250a2b81fd04424f260dbfe996cc0a4794504e0c4262efc7969db363be02c79bcf9
-  data.tar.gz: 9fd7a57fe732f18b6f568a96052ba7a5bf51423e93072ba230c7a57866b828dfe07e16b11f937c5d84d0cb01619fe027ccd80c0a0a4fb939d94a41f127ff5e3c
+  metadata.gz: 29173c2331bc0e533aa1d3aa66aa555bad7802e5fbb4207641b11aebc0a18c0d4e44ebed0b167c8fe4bddf403d2dd5a61349afeffecfc28a65359c3bb4cce26b
+  data.tar.gz: 7fdf58624182bdd4e354cad2d4f9ba001cff5a8c8c0ba02ac043607a87e2372311deb85bc0306d3aa167e98170a6049f33f3203b202ebcf25eac75e65dc23a17

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 5.1.5
+- Make retryable DataStreams creation at configure phase (#943)
+- Handle @hosts parameter on data_stream plugin (#942)
+- allow specifying custom ILM policies for data streams (#933)
+
 ### 5.1.4
 - Handle ES8 or above more strictly (#931)
 - fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)

data/README.md

@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
 data_stream_name test
 ```
 
-When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
+When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name` or `data_stream_ilm_policy`, ILM default policy is set to the specified data stream.
 Then, the matching index template is also created automatically.
 
 ### data_stream_name
@@ -1545,6 +1545,21 @@ There are some limitations about naming rule.
 
 In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
 
+
+### data_stream_ilm_policy
+
+You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
+### data_stream_ilm_policy_overwrite
+
+Specify whether the data stream ILM policy should be overwritten.
+
+Default value is `false`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.1.4'
+  s.version       = '5.1.5'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -11,6 +11,9 @@ module Fluent::Plugin
     config_param :data_stream_name, :string
     config_param :data_stream_ilm_name, :string, :default => nil
     config_param :data_stream_template_name, :string, :default => nil
+    config_param :data_stream_ilm_policy, :string, :default => nil
+    config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -29,6 +32,7 @@ module Fluent::Plugin
 
       @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
       @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+      @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
 
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
@@ -38,13 +42,16 @@ module Fluent::Plugin
         @data_stream_names = []
       end
 
-      @client = client
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
-          create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
-          create_data_stream(@data_stream_name)
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_data_stream(@data_stream_name)
+          end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
         end
@@ -76,21 +83,24 @@ module Fluent::Plugin
       end
     end
 
-    def create_ilm_policy(datastream_name, template_name, ilm_name, host)
-      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
+      unless @data_stream_ilm_policy_overwrite
+        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
+      end
+
       params = {
         policy_id: ilm_name,
-        body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
+        body: @data_stream_ilm_policy
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.xpack.ilm.put_policy(params)
+        client(host).xpack.ilm.put_policy(params)
       end
     end
 
-    def create_index_template(datastream_name, template_name, ilm_name, host)
-      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
+    def create_index_template(datastream_name, template_name, ilm_name, host = nil)
+      return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
       body = {
         "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
@@ -107,16 +117,16 @@ module Fluent::Plugin
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.put_index_template(params)
+        client(host).indices.put_index_template(params)
       end
     end
 
-    def data_stream_exist?(datastream_name)
+    def data_stream_exist?(datastream_name, host = nil)
       params = {
         name: datastream_name
       }
       begin
-        response = @client.indices.get_data_stream(params)
+        response = client(host).indices.get_data_stream(params)
         return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
       rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
         log.info "Specified data stream does not exist. Will be created: <#{e}>"
@@ -124,21 +134,21 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(datastream_name)
-      return if data_stream_exist?(datastream_name)
+    def create_data_stream(datastream_name, host = nil)
+      return if data_stream_exist?(datastream_name, host)
       params = {
         name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.create_data_stream(params)
+        client(host).indices.create_data_stream(params)
       end
     end
 
-    def ilm_policy_exists?(policy_id)
+    def ilm_policy_exists?(policy_id, host = nil)
       begin
-        @client.ilm.get_policy(policy_id: policy_id)
+        client(host).ilm.get_policy(policy_id: policy_id)
         true
       rescue
         false
@@ -192,8 +202,9 @@ module Fluent::Plugin
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
       data_stream_ilm_name = @data_stream_ilm_name
-      host = @host
+      host = nil
       if @use_placeholder
+        host = extract_placeholders(@host, chunk)
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
         data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
@@ -230,7 +241,7 @@ module Fluent::Plugin
         body: bulk_message
       }
       begin
-        response = @client.bulk(params)
+        response = client(host).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -62,56 +62,61 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
 
-  def stub_ilm_policy(name="foo_ilm_policy")
-    stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_index_template(name="foo_tpl")
-    stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_index_template(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_data_stream(name="foo")
-    stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_data_stream(name="foo", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_ilm?(name="foo_ilm_policy")
-    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_nonexistent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_nonexistent_ilm?(name="foo_ilm_policy")
-    stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_nonexistent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl")
-    stub_request(:post, "http://localhost:9200/#{datastream_name}/_bulk").with do |req|
+  def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").
+      to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
+  end
+
+  def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
+    stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
-    stub_request(:post, "http://localhost:9200/#{ilm_name}/_bulk").with do |req|
+    stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
-    stub_request(:post, "http://localhost:9200/#{template_name}/_bulk").with do |req|
+    stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
@@ -119,9 +124,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0", headers={})
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
   end
 
   def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
@@ -437,6 +442,56 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
+  def test_datastream_configure_retry
+    stub_elastic_info
+    stub_nonexistent_ilm?
+    stub_ilm_policy
+    stub_nonexistent_template_retry?
+    stub_index_template
+    stub_nonexistent_data_stream?
+    stub_data_stream
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => "foo_ilm_policy",
+      'data_stream_template_name' => "foo_tpl"
+    })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+  end
+
+  def test_hosts_list_configure
+    config = %{
+      hosts            https://john:password@host1:443/elastic/,http://host2
+      path             /default_path
+      user             default_user
+      password         default_password
+      data_stream_name default
+    }
+    stub_elastic_info("https://host1:443/elastic//", "7.9.0",
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
+    stub_elastic_info("http://host2/default_path/_data_stream/default", "7.9.0",
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
+    stub_existent_data_stream?("default", "https://host1/elastic/")
+    instance = driver(config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'password', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
   def test_existent_data_stream
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -659,4 +714,86 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
 
     assert_equal(4, connection_resets)
   end
+
+  def test_doesnt_update_ilm_policy_if_overwrite_unset
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+    }
+
+    stub_elastic_info
+    stub_index_template
+    stub_existent_data_stream?
+    stub_existent_ilm?
+    stub_data_stream
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 0)
+  end
+
+  def test_updates_ilm_policy_if_overwrite_set
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+      data_stream_ilm_policy_overwrite  true
+    }
+
+    stub_elastic_info
+    stub_index_template
+    stub_existent_data_stream?
+    stub_existent_ilm?
+    stub_data_stream
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+      body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
+      times: 1)
+  end
+
+  def test_creates_custom_ilm_policy_if_none_exists
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                  foo
+      data_stream_ilm_name              foo_ilm_policy
+      data_stream_ilm_policy            {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
+    }
+
+    stub_elastic_info
+    stub_index_template("foo_template")
+    stub_data_stream
+    stub_nonexistent_data_stream?
+    stub_nonexistent_ilm?
+    stub_nonexistent_template?("foo_template")
+
+    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    assert_nothing_raised {
+      driver(config)
+    }
+
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+      body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
+      times: 1)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.1.4
+  version: 5.1.5
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-22 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd