fluent-plugin-elasticsearch 1.16.0 → 1.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15e80a669ab0e24b83a279e1cbcc3240b76560b445319b7af7b2f719523b86e1
-  data.tar.gz: df01b4f813ffeba65619b778d418384b1cf3df3323929cf2921cf1273a336738
+  metadata.gz: 222baba6d90b4d1146642683a3e6144b6339d2c4dc6a6a456c238fc984ae6023
+  data.tar.gz: 929e1d360821206f8a17e76e2a4948d65c855c77ab68fcca372a2c9be2471170
 SHA512:
-  metadata.gz: 2e44489b1f603ae61d9566c4cd0d6c16a7cec6ee5c701053f884ee284945fa2f48407674108223ec6c4c7127daabcaf17b6eb534b06d8c26339dd3aaba2fa908
-  data.tar.gz: cee4380e5d0aecfd9305bdbc68eaf3c2dbc9fef6b0e8f88f05ad4870a33d3b330983118a6ebe492ac4645805ec23c7acc270b91d37d6a222ed03ebc5ae28c6a5
+  metadata.gz: 0aac8b9d854aa8042f8fd0a79faec608e202ae77fc53e8ad123f28b9b38021fd6de5cc011427cf1311bc19d701c4c529746647fa007579303fb13018e99f0d51
+  data.tar.gz: f5622800b63e568c2b8e15fe4e6d05523bede040d9d86edc5f88db557d4eff4c66223fc3beeac3b687af4e9680e25972ca21e8b572012b5a2f7a3b1726a7b453

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 1.16.1
+- allow configure of retry_tag so messages can be routed through a different pipeline (#419)
+- fix #417. emit_error_event using an exception (#418)
+
 ### 1.16.0
 - evaluate bulk request failures and reroute failed messages (#405)
 

data/README.md

@@ -45,6 +45,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [remove_keys](#remove_keys)
   + [remove_keys_on_update](#remove_keys_on_update)
   + [remove_keys_on_update_key](#remove_keys_on_update_key)
+  + [retry_tag](#retry_tag)
   + [write_operation](#write_operation)
   + [time_parse_error_tag](#time_parse_error_tag)
   + [reconnect_on_error](#reconnect_on_error)
@@ -435,6 +436,21 @@ present in the record then the keys in record are used, if the `remove_keys_on_u
 remove_keys_on_update_key keys_to_skip
 ```
 
+### retry_tag
+
+This setting allows custom routing of messages in response to bulk request failures.  The default behavior is to emit
+failed records using the same tag that was provided.  When set to a value other then `nil`, failed messages are emitted
+with the specified tag:
+
+```
+retry_tag 'retry_es'
+```
+**NOTE:** `retry_tag` is optional. If you would rather use labels to reroute retries, add a label (e.g '@label @SOMELABEL') to your fluent
+elasticsearch plugin configuration. Retry records are, by default, submitted for retry to the ROOT label, which means
+records will flow through your fluentd pipeline from the beginning.  This may nor may not be a problem if the pipeline
+is idempotent - that is - you can process a record again with no changes.  Use tagging or labeling to ensure your retry
+records are not processed again by your fluentd processing pipeline.
+
 ### write_operation
 
 The write_operation can be any of:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.16.0'
+  s.version       = '1.16.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -61,7 +61,7 @@ class Fluent::ElasticsearchErrorHandler
         stats[:duplicates] += 1
       when 400 == status
         stats[:bad_argument] += 1
-        @plugin.router.emit_error_event(tag, time, rawrecord, '400 - Rejected by Elasticsearch')
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
       else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
@@ -71,7 +71,7 @@ class Fluent::ElasticsearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
           stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, "#{status} - No error type provided in the response")
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
           next
         end
         stats[type] += 1

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -56,6 +56,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :request_timeout, :time, :default => 5
   config_param :reload_connections, :bool, :default => true
   config_param :reload_on_failure, :bool, :default => false
+  config_param :retry_tag, :string, :default=>nil
   config_param :resurrect_after, :time, :default => 60
   config_param :time_key, :string, :default => nil
   config_param :time_key_exclude_timestamp, :bool, :default => false
@@ -428,7 +429,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
         error.handle_error(response, tag, chunk, bulk_message_count) 
       end
     rescue RetryStreamError => e
-      router.emit_stream(tag, e.retry_stream)
+      emit_tag = @retry_tag ? @retry_tag : tag
+      router.emit_stream(emit_tag, e.retry_stream)
     rescue *client.transport.host_unreachable_exceptions => e
       if retries < 2
         retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -11,7 +11,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     def initialize(log)
       @log = log
       @write_operation = 'index'
-      @error_events = Fluent::MultiEventStream.new
+      @error_events = []
     end
 
     def router
@@ -19,7 +19,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
 
     def emit_error_event(tag, time, record, e)
-       @error_events.add(time, record)
+       @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
     def process_message(tag, meta, header, time, record, bulk_message)
@@ -75,7 +75,8 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
      }))
     chunk = MockChunk.new(records)
     @handler.handle_error(response, 'atag', chunk, records.length)
-    assert_equal(1, @plugin.error_events.instance_variable_get(:@time_array).size)
+    assert_equal(1, @plugin.error_events.size)
+    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
 
   def test_retry_error
@@ -181,10 +182,13 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       assert_equal 2, records[0]['_id']
       assert_equal 6, records[1]['_id']
       assert_equal 8, records[2]['_id']
-      errors = @plugin.error_events.collect {|time, record| record}
-      assert_equal 2, errors.length
-      assert_equal 5, errors[0]['_id']
-      assert_equal 7, errors[1]['_id']
+      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
+      assert_equal 2, error_ids.length
+      assert_equal 5, error_ids[0]
+      assert_equal 7, error_ids[1]
+      @plugin.error_events.collect {|h| h[:error]}.each do |e|
+        assert_true e.respond_to?(:backtrace)
+      end
     end
     assert_true failed
 

data/test/plugin/test_out_elasticsearch.rb

@@ -1348,6 +1348,39 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
+  def test_bulk_error_retags_when_configured
+    driver.configure("retry_tag retry\n")
+    stub_elastic_ping
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            }
+           ]
+        })
+     }
+    end)
+    driver.emit(sample_record, 1)
+
+    driver.expect_emit('retry', 1, sample_record)
+    driver.run
+  end
+
   def test_bulk_error
     stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.16.0
+  version: 1.16.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-02 00:00:00.000000000 Z
+date: 2018-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd