fluent-plugin-elasticsearch 2.10.0 → 2.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f778421b8290ea1db16ad57f1c0be9a35e656d201e5b52c502ae9301c60a67c3
-  data.tar.gz: 7e72d88f792ac7a959faa393e91fcc08917ada672860730352219b5d26b167b4
+  metadata.gz: 8e64450e9c4dca90c39c52e6c9bd2a6b3bc3958644f608fde7444c2024f68b62
+  data.tar.gz: 05a52be8643cacf7e5301d50a6d545c7105acd84ffca1ba417551929d4c0fed4
 SHA512:
-  metadata.gz: 8222728bf41a377029aa620f94f7d773529d4dbae47081158022c49b8ebb08aed224d93208db1d4933663ea5226decd1e99096c9890a0f59a58d22e8aca9d5ae
-  data.tar.gz: fe857ad374b96cfc4eeedb02642fc1bebfb4da43f0cba397789db9f74a6df6704ffb1c57b65524b00866e4b540c97130c387f6252bbf35fa1215d95c7f3c1f1a
+  metadata.gz: 3e6c52a7f807205e3f851039710b207f0afccdafd3fe9337a471e65752bc9aa8a5737790b33d0dc2e1a09569cad282cdc97450e3cffb23bef16907b9680cf521
+  data.tar.gz: af806e72ce8af92c9486432feca7d2a5fc1f5c241d98de54af6301321864fb29a88c2a81be1c487bb81292947d9d970df10d95b6b683434910b0c52b82bbb589

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 2.10.1
+- Uplift Merge pull request #419 from jcantrill/retry_prefix (#421)
+- Uplift Merge pull request #418 from jcantrill/emit_exception (#420)
+
 ### 2.10.0
 - Uplift Merge pull request #405 from jcantrill/sanitize_bulk (#414)
 

data/README.md

@@ -47,6 +47,7 @@ Current maintainers: @cosmo0920
   + [remove_keys](#remove_keys)
   + [remove_keys_on_update](#remove_keys_on_update)
   + [remove_keys_on_update_key](#remove_keys_on_update_key)
+  + [retry_tag](#retry_tag)
   + [write_operation](#write_operation)
   + [time_parse_error_tag](#time_parse_error_tag)
   + [reconnect_on_error](#reconnect_on_error)
@@ -526,6 +527,21 @@ present in the record then the keys in record are used, if the `remove_keys_on_u
 remove_keys_on_update_key keys_to_skip
 ```
 
+### retry_tag
+
+This setting allows custom routing of messages in response to bulk request failures.  The default behavior is to emit
+failed records using the same tag that was provided.  When set to a value other then `nil`, failed messages are emitted
+with the specified tag:
+
+```
+retry_tag 'retry_es'
+```
+**NOTE:** `retry_tag` is optional. If you would rather use labels to reroute retries, add a label (e.g '@label @SOMELABEL') to your fluent
+elasticsearch plugin configuration. Retry records are, by default, submitted for retry to the ROOT label, which means
+records will flow through your fluentd pipeline from the beginning.  This may nor may not be a problem if the pipeline
+is idempotent - that is - you can process a record again with no changes.  Use tagging or labeling to ensure your retry
+records are not processed again by your fluentd processing pipeline.
+
 ### write_operation
 
 The write_operation can be any of:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.10.0'
+  s.version       = '2.10.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -61,7 +61,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         stats[:duplicates] += 1
       when 400 == status
         stats[:bad_argument] += 1
-        @plugin.router.emit_error_event(tag, time, rawrecord, '400 - Rejected by Elasticsearch')
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
       else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
@@ -71,7 +71,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
           stats[:errors_bad_resp] += 1
-          @plugin.router.emit_error_event(tag, time, rawrecord, "#{status} - No error type provided in the response")
+          @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
           next
         end
         stats[type] += 1

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -67,6 +67,7 @@ EOC
     config_param :request_timeout, :time, :default => 5
     config_param :reload_connections, :bool, :default => true
     config_param :reload_on_failure, :bool, :default => false
+    config_param :retry_tag, :string, :default=>nil
     config_param :resurrect_after, :time, :default => 60
     config_param :time_key, :string, :default => nil
     config_param :time_key_exclude_timestamp, :bool, :default => false
@@ -510,7 +511,8 @@ EOC
           error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
         end
       rescue RetryStreamError => e
-        router.emit_stream(tag, e.retry_stream)
+        emit_tag = @retry_tag ? @retry_tag : tag
+        router.emit_stream(emit_tag, e.retry_stream)
       rescue *client.transport.host_unreachable_exceptions => e
         if retries < 2
           retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -11,7 +11,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     def initialize(log)
       @log = log
       @write_operation = 'index'
-      @error_events = Fluent::MultiEventStream.new
+      @error_events = []
     end
 
     def router
@@ -19,7 +19,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
 
     def emit_error_event(tag, time, record, e)
-       @error_events.add(time, record)
+       @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
     def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
@@ -72,7 +72,8 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     chunk = MockChunk.new(records)
     dummy_extracted_values = []
     @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
-    assert_equal(1, @plugin.error_events.instance_variable_get(:@time_array).size)
+    assert_equal(1, @plugin.error_events.size)
+    assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
 
   def test_retry_error
@@ -179,10 +180,13 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       assert_equal 2, records[0]['_id']
       assert_equal 6, records[1]['_id']
       assert_equal 8, records[2]['_id']
-      errors = @plugin.error_events.collect {|time, record| record}
-      assert_equal 2, errors.length
-      assert_equal 5, errors[0]['_id']
-      assert_equal 7, errors[1]['_id']
+      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
+      assert_equal 2, error_ids.length
+      assert_equal 5, error_ids[0]
+      assert_equal 7, error_ids[1]
+      @plugin.error_events.collect {|h| h[:error]}.each do |e|
+        assert_true e.respond_to?(:backtrace)
+      end
     end
     assert_true failed
 

data/test/plugin/test_out_elasticsearch.rb

@@ -1718,6 +1718,40 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
+  def test_bulk_error_retags_when_configured
+    driver.configure("retry_tag retry\n")
+    stub_elastic_ping
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            }
+           ]
+        })
+     }
+    end)
+
+    driver.run(default_tag: 'test') do
+      driver.feed(1, sample_record)
+    end
+
+    assert_equal [['retry', 1, sample_record]], driver.events
+  end
 
   def test_bulk_error
     stub_elastic_ping

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.10.0
+  version: 2.10.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2018-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd