fluent-plugin-elasticsearch 3.3.2 → 3.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65a43866400112b8742307709fb258518efdee379cdbb56b324ed365d8a4e1d7
-  data.tar.gz: 716149f2c5aa5380bbfc95e916d730572546bfae2520ed0b792dc6b6a828d78c
+  metadata.gz: 2d075f2b8fe2deb8579ce232104c2e497eb6aee6703b28f0288728b2f2622c23
+  data.tar.gz: 580c94f146d2039d242cc0e02487bc771e12e0f9ac885d634ea4fc169c6e577b
 SHA512:
-  metadata.gz: 6d98dc4fef7b9fe208d549fc93f8a4809bfb56f156bee069bf0f599891d13b29adad5d119e9e7a2274f31aede5a13879dbf16223f98a3642d598f9f71eb2665f
-  data.tar.gz: ab08e35c256740009631cf5d8fa13c9f8ab9edc98f172e106d111177b4a069db13ba27a60f201c71411727bdb9cac63de716d0b6d9e4ae2184a3eac8ae12131f
+  metadata.gz: d98b07c8fe8e0c13d290e50957c9f2f6f6d988173bc3dbe3a6f36bedf546531f85639d3fce637295eefb14d5c3726a77e3b91fd418b451080caae03399177df8
+  data.tar.gz: 3cc3ab52cb741c020568fd55e82d496805b5a66b2d54ea6a9cf09917413099934f17bc6d69fce8b9ef431f7b0dde6662d27d9649d6ecd4330dce55fae932364d

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 3.3.3
+- Add unit test for exception message (#563)
+- Add ignore_exceptions config (#562)
+
 ### 3.3.2
 - Fix support for host, hosts placeholders (#560)
 - typo fixes in README.md (#559)

data/README.md

@@ -82,6 +82,8 @@ Current maintainers: @cosmo0920
   + [Multi workers](#multi-workers)
   + [log_es_400_reason](#log_es_400_reason)
   + [suppress_doc_wrap](#suppress_doc_wrap)
+  + [ignore_exceptions](#ignore_exceptions)
+  + [exception_backup](#exception_backup)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -1032,6 +1034,24 @@ By default, record body is wrapped by 'doc'. This behavior can not handle update
 
 Default value is `false`.
 
+## ignore_exceptions
+
+A list of exception that will be ignored - when the exception occurs the chunk will be discarded and the buffer retry mechanism won't be called. It is possible also to specify classes at higher level in the hierarchy. For example
+
+```
+ignore_exceptions ["Elasticsearch::Transport::Transport::ServerError"]
+```
+
+will match all subclasses of `ServerError` - `Elasticsearch::Transport::Transport::Errors::BadRequest`, `Elasticsearch::Transport::Transport::Errors::ServiceUnavailable`, etc.
+
+Default value is empty list (no exception is ignored).
+
+## exception_backup
+
+Indicates whether to backup chunk when ignore exception occurs.
+
+Default value is `true`.
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.3.2'
+  s.version       = '3.3.3'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -133,6 +133,8 @@ EOC
     config_param :log_es_400_reason, :bool, :default => false
     config_param :custom_headers, :hash, :default => {}
     config_param :suppress_doc_wrap, :bool, :default => false
+    config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
+    config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -275,6 +277,16 @@ EOC
 
       @routing_key_name = configure_routing_key_name
       @current_config = nil
+
+      @ignore_exception_classes = @ignore_exceptions.map do |exception|
+        unless Object.const_defined?(exception)
+          log.warn "Cannot find class #{exception}. Will ignore it."
+
+          nil
+        else
+          Object.const_get(exception)
+        end
+      end.compact
     end
 
     def backend_options
@@ -678,10 +690,17 @@ EOC
         emit_tag = @retry_tag ? @retry_tag : tag
         router.emit_stream(emit_tag, e.retry_stream)
       rescue => e
+        ignore = @ignore_exception_classes.any? { |clazz| e.class <= clazz }
+
+        log.warn "Exception ignored in tag #{tag}: #{e.class.name} #{e.message}" if ignore
+
         @_es = nil if @reconnect_on_error
         @_es_info = nil if @reconnect_on_error
+
+        raise UnrecoverableRequestFailure if ignore && @exception_backup
+
         # FIXME: identify unrecoverable errors and raise UnrecoverableRequestFailure instead
-        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{connection_options_description(info.host)}): #{e.message}"
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{connection_options_description(info.host)}): #{e.message}" unless ignore
       end
     end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -1546,6 +1546,26 @@ class ElasticsearchOutput < Test::Unit::TestCase
       assert_requested(first_request)
       assert_requested(second_request)
     end
+
+    def test_writes_to_extracted_host_with_placeholder_replaced_in_exception_message
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'host' => 'myhost-${pipeline_id}',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.iso8601
+      pipeline_id = "1"
+      request = stub_elastic_unavailable("http://myhost-1:9200/_bulk")
+      exception = assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
+        driver.run(default_tag: 'test') do
+          driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
+        end
+      }
+      assert_equal("could not push logs to Elasticsearch cluster ({:host=>\"myhost-1\", :port=>9200, :scheme=>\"http\"}): [503] ", exception.message)
+    end
   end
 
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
@@ -2569,4 +2589,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
     )
   end
 
+  def test_ignore_exception
+    driver.configure('ignore_exceptions ["Elasticsearch::Transport::Transport::Errors::ServiceUnavailable"]')
+    stub_elastic_unavailable
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+  end
+
+  def test_ignore_exception_with_superclass
+    driver.configure('ignore_exceptions ["Elasticsearch::Transport::Transport::ServerError"]')
+    stub_elastic_unavailable
+
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+  end
+
+  def test_ignore_excetion_handles_appropriate_ones
+    driver.configure('ignore_exceptions ["Faraday::ConnectionFailed"]')
+    stub_elastic_unavailable
+
+    assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
+    }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.3.3
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2019-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd