fluent-plugin-elasticsearch 5.0.5 → 5.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e7a5cd238a268dd3f141ce4ad1dedfacbacc98fdd22f73839bb0e1cdbccc586
-  data.tar.gz: 8077a55fae311fd0bd9cd84fc2b2eb34fad8f7a2a890b2707d37c7ac1c753975
+  metadata.gz: 6ad91e234f95a4d2536ac1de2630c6af572e249f5b1ff9a40a83d427f8901413
+  data.tar.gz: 91fbf4c3d8fe17e05094f5fa991ec08ee5491a528d39d3d048678adfc30f1192
 SHA512:
-  metadata.gz: 7bede6c6213db75e2636c6bf31e65b3256ae709ef600b9f3fca8973943344e07421daf4ed04fc0f3dd06f6b1f662c93c27d9240a323f8139aee3bf2a99ed0f9b
-  data.tar.gz: caed0a434434c903451cdd4b92ae4427ca39984a8405154308d70a2b68351c6a05b7e505c08cd776230cef9dbbda46f4256cd7de138f3549a1860b5b6cee8e3c
+  metadata.gz: 531ba1b0b8824894925a502eb310ea7b9288314dbe940bcee1190f60abea5f9d398c1b815ad24f47cc213342e93d87e1ddc74172f7a8ca931ea59da5527f5060
+  data.tar.gz: c8820ca2ae83a2b9082b35f806669cd5783b3ee6cb3e4baa960a937cb2a2b5554b6665bfe9dea666bfd9b5b6b2f462fc7147d9f28ccf4d2281ba4620de0f4acf

data/History.md

@@ -2,6 +2,21 @@
 
 ### [Unreleased]
 
+### 5.1.3
+-  fixing execution order for dynamic data stream creation (#928)
+
+### 5.1.2
+-  Fix default values of datastream parameters (#926)
+
+### 5.1.1
+- Report appropriate error for data_stream parameters (#922)
+- Add ILM and template parameters for data streams (#920)
+- Support Buffer in Data Stream Output (#917)
+
+### 5.1.0
+- Correct default target bytes value (#914)
+- Handle elasticsearch-ruby 7.14 properly (#913)
+
 ### 5.0.5
 - Drop json_parse_exception messages for bulk failures (#900)
 - GitHub Actions: Drop Ruby 2.5 due to EOL (#894)

data/README.md

@@ -11,7 +11,7 @@ Send your logs to Elasticsearch (and search them with Kibana maybe?)
 
 Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service)
 
-Current maintainers: @cosmo0920
+Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920), [Kentaro Hayashi | @kenhys](https://github.com/kenhys)
 
 * [Installation](#installation)
 * [Usage](#usage)
@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
 data_stream_name test
 ```
 
-When `@type elasticsearch_data_stream` is used, ILM default policy is set to the specified data stream.
+When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
 Then, the matching index template is also created automatically.
 
 ### data_stream_name
@@ -1529,6 +1529,18 @@ Then, the matching index template is also created automatically.
 You can specify Elasticsearch data stream name by this parameter.
 This parameter is mandatory for `elasticsearch_data_stream`.
 
+### data_stream_template_name
+
+You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template. 
+
+Default value is `data_stream_name`.
+
+### data_stream_ilm_name
+
+You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless `data_stream_template_name` is defined, in that case the ILM will be set to the one specified in the matching index template).
+
+Default value is `data_stream_name`.
+
 There are some limitations about naming rule.
 
 In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.0.5'
+  s.version       = '5.1.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -32,13 +32,25 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
+  def host_unreachable_exceptions
+    if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+      # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
+      # Elasticsearch class on Transport.
+      # This is why #host_unreachable_exceptions is not callable directly
+      # via transport (not transport's transport instance accessor) any more.
+      client.transport.transport.host_unreachable_exceptions
+    else
+      client.transport.host_unreachable_exceptions
+    end
+  end
+
   def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
     transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -72,7 +72,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
-    TARGET_BULK_BYTES = 20 * 1024 * 1024
+    DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
     config_param :host, :string,  :default => 'localhost'
@@ -166,7 +166,7 @@ EOC
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
-    config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
+    config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
     config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -1,3 +1,4 @@
+
 require_relative 'out_elasticsearch'
 
 module Fluent::Plugin
@@ -8,6 +9,8 @@ module Fluent::Plugin
     helpers :event_emitter
 
     config_param :data_stream_name, :string
+    config_param :data_stream_ilm_name, :string, :default => nil
+    config_param :data_stream_template_name, :string, :default => nil
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
 
@@ -24,9 +27,12 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
       end
 
+      @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+
       # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
       unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
-        validate_data_stream_name
+        validate_data_stream_parameters
       else
         @use_placeholder = true
         @data_stream_names = []
@@ -36,8 +42,8 @@ module Fluent::Plugin
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_ilm_policy(@data_stream_name)
-          create_index_template(@data_stream_name)
+          create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
+          create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
           create_data_stream(@data_stream_name)
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -45,31 +51,35 @@ module Fluent::Plugin
       end
     end
 
-    def validate_data_stream_name
-      unless valid_data_stream_name?
-        unless start_with_valid_characters?
-          if not_dots?
-            raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
-          else
-            raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name,
+       "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
           end
-        end
-        unless valid_characters?
-          raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
-        end
-        unless lowercase_only?
-          raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
-        end
-        if @data_stream_name.bytes.size > 255
-          raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
         end
       end
     end
 
-    def create_ilm_policy(name)
-      return if data_stream_exist?(name)
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
       params = {
-        policy_id: "#{name}_policy",
+        policy_id: "#{ilm_name}_policy",
         body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
       }
       retry_operate(@max_retry_putting_template,
@@ -79,19 +89,19 @@ module Fluent::Plugin
       end
     end
 
-    def create_index_template(name)
-      return if data_stream_exist?(name)
+    def create_index_template(datastream_name, template_name, ilm_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
       body = {
-        "index_patterns" => ["#{name}*"],
+        "index_patterns" => ["#{datastream_name}*"],
         "data_stream" => {},
         "template" => {
           "settings" => {
-            "index.lifecycle.name" => "#{name}_policy"
+            "index.lifecycle.name" => "#{ilm_name}"
           }
         }
       }
       params = {
-        name: name,
+        name: template_name,
         body: body
       }
       retry_operate(@max_retry_putting_template,
@@ -101,9 +111,9 @@ module Fluent::Plugin
       end
     end
 
-    def data_stream_exist?(name)
+    def data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       begin
         response = @client.indices.get_data_stream(params)
@@ -114,10 +124,10 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(name)
-      return if data_stream_exist?(name)
+    def create_data_stream(datastream_name)
+      return if data_stream_exist?(datastream_name)
       params = {
-        "name": name
+        name: datastream_name
       }
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
@@ -126,28 +136,48 @@ module Fluent::Plugin
       end
     end
 
-    def valid_data_stream_name?
-      lowercase_only? and
-        valid_characters? and
-        start_with_valid_characters? and
-        not_dots? and
-        @data_stream_name.bytes.size <= 255
+    def ilm_policy_exists?(policy_id)
+      begin
+        @client.ilm.get_policy(policy_id: policy_id)
+        true
+      rescue
+        false
+      end
+    end
+
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
     end
 
-    def lowercase_only?
-      @data_stream_name.downcase == @data_stream_name
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
     end
 
-    def valid_characters?
-      not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
     end
 
-    def start_with_valid_characters?
-      not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
     end
 
-    def not_dots?
-      not (@data_stream_name == "." or @data_stream_name == "..")
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
     end
 
     def client_library_version
@@ -160,12 +190,17 @@ module Fluent::Plugin
 
     def write(chunk)
       data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      data_stream_ilm_name = @data_stream_ilm_name
+      host = @host
       if @use_placeholder
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
         unless @data_stream_names.include?(data_stream_name)
           begin
-            create_ilm_policy(data_stream_name)
-            create_index_template(data_stream_name)
+            create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
             create_data_stream(data_stream_name)
             @data_stream_names << data_stream_name
           rescue => e
@@ -200,7 +235,7 @@ module Fluent::Plugin
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end
       rescue => e
-        log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
       end
     end
 

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -19,7 +19,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   end
 
   def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
   end
 

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -27,9 +27,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     assert_equal(expected, xpack_info)
   end
@@ -37,18 +43,21 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
       to_return(:status => 404, :body => "", :headers => {})
+    stub_elastic_info
     assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
   end
 
@@ -59,6 +68,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     create_ilm_policy("fluent-policy")
 
     assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)

data/test/plugin/test_in_elasticsearch.rb

@@ -31,6 +31,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def sample_response(index_name="fluentd")
     {
       "took"=>4,
@@ -322,6 +327,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG)
     driver.run(expect_emits: 1, timeout: 10)
@@ -337,6 +343,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
     driver.run(expect_emits: 1, timeout: 10)
@@ -352,6 +359,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
     driver.run(expect_emits: 1, timeout: 10)
@@ -370,6 +378,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
                       timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
@@ -389,6 +398,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
     driver.run(expect_emits: 1, timeout: 10)
@@ -412,6 +422,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
     driver.run(expect_emits: 1, timeout: 10)
@@ -434,6 +445,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1
     end
+    stub_elastic_info
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,