fluent-plugin-elasticsearch 1.15.1 → 1.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00a9fd19dd2727f3b8923ba7ecab25944afdcc52bc1e5b3f12571f95626640d8
-  data.tar.gz: ff12458ed88da30d28a1256105c4c2b90e02706b30b691c63f6d257dcc12977c
+  metadata.gz: dc1ee6d297a64d46320b2b48b8528faf9a377f1a9af9274c083f36e0d7f48c88
+  data.tar.gz: ff6de82c88a71a3f8a3c47951fadf0b555fe4118feb18fe911c4ef7b66178218
 SHA512:
-  metadata.gz: d6f158b0d213b1bea7efc8f7ca2c85cc07ffe92195692d2c8fe3b6a9ede345e02870033213894ccd1da8afc38df08535c92d7ef336ae0ef4ef1319e4ea9eef8b
-  data.tar.gz: 3f8b3b2537b3d72017f9ca85493c97997fd4770937f8bd158836b9de7e48a7053a03abf42264fae0a64d56a5600f3f24c9a1edd4d918cf0d71b71377d5895d76
+  metadata.gz: 3123a75aeee15020ec775457d36bfe6eaf52ba22309163c37859ca096556605dc400a98495f587f954e2e7b4d4ad2aa9cf9b7010c9c75158d911feaa6e181fa9
+  data.tar.gz: 506fc86e5930c18c38992d477d7865363bd351851e141879c67834dca05828f710795fc63725a6f5bc7fde9732660fa97a5caec45d6361b6c974e391fa1ef36c

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.15.2
+- handle case where stats not processed in order; add testing (#410)
+
 ### 1.15.1
 - successful operation if all duplicates (#406)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.15.1'
+  s.version       = '1.15.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -53,29 +53,24 @@ class Fluent::ElasticsearchErrorHandler
         stats[type] += 1
       end
     end
-    if stats[:errors_bad_resp] > 0
-      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
-      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
-    end
     @plugin.log.on_debug do
       msg = ["Indexed (op = #{@plugin.write_operation})"]
       stats.each_pair { |key, value| msg << "#{value} #{key}" }
       @plugin.log.debug msg.join(', ')
     end
-    if stats[:successes] + stats[:duplicates] == bulk_message_count
-      @plugin.log.debug("retry succeeded - all #{bulk_message_count} records were successfully sent")
-      return
-    end
-    stats.each_key do |key|
-      case key
-      when 'out_of_memory_error'
-        raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
-      when 'es_rejected_execution_exception'
-        raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
-      else
-        @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
-        raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
-      end
+    case
+    when stats[:errors_bad_resp] > 0
+      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
+      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
+    when stats[:successes] + stats[:duplicates] == bulk_message_count
+      @plugin.log.info("retry succeeded - successes=#{stats[:successes]} duplicates=#{stats[:duplicates]}")
+    when stats['es_rejected_execution_exception'] > 0
+      raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
+    when stats['out_of_memory_error'] > 0
+      raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
+    else
+      @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
+      raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
     end
   end
 end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -6,20 +6,25 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
+    attr_reader :write_operation
     def initialize(log)
       @log = log
-    end
-
-    def write_operation
-      'index'
+      @write_operation = 'index'
     end
   end
 
   def setup
     Fluent::Test.setup
-    @log = Fluent::Engine.log
-    plugin = TestPlugin.new(@log)
-    @handler =  Fluent::ElasticsearchErrorHandler.new(plugin)
+    @log_device = Fluent::Test::DummyLogDevice.new
+    if defined?(ServerEngine::DaemonLogger)
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+    else
+      @log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_INFO)
+    end
+    @plugin = TestPlugin.new(@log)
+    @handler = Fluent::ElasticsearchErrorHandler.new(@plugin)
   end
 
   def parse_response(value)
@@ -39,7 +44,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         },
@@ -51,7 +56,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         },
@@ -79,7 +84,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 400,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         }
@@ -119,4 +124,171 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   end
 
+  def test_retry_with_successes_and_duplicates
+    response = parse_response(%(
+      {
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409,
+            "error" : {
+              "reason":"duplicate ID"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        }
+      ]
+      }
+    ))
+
+    @plugin.instance_variable_set(:@write_operation, 'create')
+    @handler.instance_variable_set(:@bulk_message_count, 2)
+    @handler.handle_error(response)
+    assert_match /retry succeeded - successes=1 duplicates=1/, @log.out.logs[0]
+  end
+
+  def test_bulk_rejection_errors
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 429,
+            "error" : {
+              "type" : "es_rejected_execution_exception",
+              "reason":"Elasticsearch could not process bulk index request"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::ElasticsearchErrorHandler::BulkIndexQueueFull do
+        @handler.handle_error(response)
+    end
+
+  end
+
+  def test_out_of_memory_errors
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 400,
+            "error" : {
+              "type" : "out_of_memory_error",
+              "reason":"Elasticsearch exhausted its heap"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchOutOfMemory do
+        @handler.handle_error(response)
+    end
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.15.1
+  version: 1.15.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-22 00:00:00.000000000 Z
+date: 2018-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd