fluent-plugin-elasticsearch 2.9.1 → 2.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68bc9bdc4c7c5e52553aafacc5b7018e1cc7f5b01e4f5bcacb112443787b4ced
-  data.tar.gz: 9bd3383b8b79e480f86753827c88fc127767357e02729dec6e7d1b8a676fd0de
+  metadata.gz: c6294313d2bbda7a0933074c22b9d94db7d0a6ae7e7518210a0b56fa6a5ae2a2
+  data.tar.gz: a94f6a5ce9d4e79c3280d512cb8acd9174aac263179f09f18088f50f6a1f9a62
 SHA512:
-  metadata.gz: 44987ff76928eb5e78607820ef84ad4a9ea5dc8d8c8043afff8506919ac87cc4cf6643c9a3ff875d412c3a82012bcbef32dda27fcfe61d4a87bb33e8de63d9dc
-  data.tar.gz: 20169754c661e2de4a938d208c34e10adbde571790082a28c2c7f76996c5c0a5db5b74edcf1642a352f2a0196334706e575bb2b9ec46e8dd55c5ff1d695a5460
+  metadata.gz: d0184d1b690d6c9118bcb419e6904411b61dd8cac289f59e78ba554fbe3c953ac7b697b5271a491fbcf5b391d5691a7c3982288ed10bf6b9e1eca54a705450cb
+  data.tar.gz: e0efbbd71009eaa75e8372677fbd169a54f1d143b2ce12df112c075af8b523a18650472ea6c4692267ca39cb48f30204d81dbd556bc18187d2c31a7c515ee4a8

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 2.9.2
+- Uplift Merge pull request #410 from richm/v0.12-consistent-errors-and-tests (#411)
+- Add correct link for include_timestamp (#408)
+
 ### 2.9.1
 - Uplift Merge pull request #406 from richm/v0.12-successes-duplicates-no-retry (#407)
 

data/README.md

@@ -28,7 +28,7 @@ Current maintainers: @cosmo0920
   + [time_precision](#time_precision)
   + [time_key](#time_key)
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
-  + [include_timestamp](#time_key_exclude_timestamp)
+  + [include_timestamp](#include_timestamp)
   + [utc_index](#utc_index)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.9.1'
+  s.version       = '2.9.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -53,29 +53,24 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         stats[type] += 1
       end
     end
-    if stats[:errors_bad_resp] > 0
-      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
-      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
-    end
     @plugin.log.on_debug do
       msg = ["Indexed (op = #{@plugin.write_operation})"]
       stats.each_pair { |key, value| msg << "#{value} #{key}" }
       @plugin.log.debug msg.join(', ')
     end
-    if stats[:successes] + stats[:duplicates] == bulk_message_count
-      @plugin.log.debug("retry succeeded - all #{bulk_message_count} records were successfully sent")
-      return
-    end
-    stats.each_key do |key|
-      case key
-      when 'out_of_memory_error'
-        raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
-      when 'es_rejected_execution_exception'
-        raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
-      else
-        @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
-        raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
-      end
+    case
+    when stats[:errors_bad_resp] > 0
+      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
+      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
+    when stats[:successes] + stats[:duplicates] == bulk_message_count
+      @plugin.log.info("retry succeeded - successes=#{stats[:successes]} duplicates=#{stats[:duplicates]}")
+    when stats['es_rejected_execution_exception'] > 0
+      raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
+    when stats['out_of_memory_error'] > 0
+      raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
+    else
+      @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
+      raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
     end
   end
 end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -6,20 +6,21 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
+    attr_reader :write_operation
     def initialize(log)
       @log = log
-    end
-
-    def write_operation
-      'index'
+      @write_operation = 'index'
     end
   end
 
   def setup
     Fluent::Test.setup
-    @log = Fluent::Engine.log
-    plugin = TestPlugin.new(@log)
-    @handler =  Fluent::Plugin::ElasticsearchErrorHandler.new(plugin)
+    @log_device = Fluent::Test::DummyLogDevice.new
+    dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+    logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+    @log = Fluent::Log.new(logger)
+    @plugin = TestPlugin.new(@log)
+    @handler = Fluent::Plugin::ElasticsearchErrorHandler.new(@plugin)
   end
 
   def parse_response(value)
@@ -39,7 +40,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         },
@@ -51,7 +52,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         },
@@ -79,7 +80,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "status" : 400,
             "error" : {
               "type" : "some unrecognized type",
-              "reason":"some error to cause version mismatch"
+              "reason":"unrecognized error"
             }
           }
         }
@@ -119,4 +120,171 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   end
 
+  def test_retry_with_successes_and_duplicates
+    response = parse_response(%(
+      {
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409,
+            "error" : {
+              "reason":"duplicate ID"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        }
+      ]
+      }
+    ))
+
+    @plugin.instance_variable_set(:@write_operation, 'create')
+    @handler.instance_variable_set(:@bulk_message_count, 2)
+    @handler.handle_error(response)
+    assert_match /retry succeeded - successes=1 duplicates=1/, @log.out.logs[0]
+  end
+
+  def test_bulk_rejection_errors
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 429,
+            "error" : {
+              "type" : "es_rejected_execution_exception",
+              "reason":"Elasticsearch could not process bulk index request"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull do
+        @handler.handle_error(response)
+    end
+
+  end
+
+  def test_out_of_memory_errors
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 400,
+            "error" : {
+              "type" : "out_of_memory_error",
+              "reason":"Elasticsearch exhausted its heap"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchOutOfMemory do
+        @handler.handle_error(response)
+    end
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.9.1
+  version: 2.9.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-23 00:00:00.000000000 Z
+date: 2018-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd