fluent-plugin-elasticsearch 2.11.9 → 2.11.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85a3883a0d6e6ca5ba98a945cd6a786e2aa9e6f65e4869809e4c8ccae5f25c1d
-  data.tar.gz: 29dec4932b60be332820bb039329c173102d4f7c4e292c3ef7d4092c992d422c
+  metadata.gz: c1cebf3b8220407620d42104808fd3a1d7a498d4a5cbfe51c0711090bcfd0cbd
+  data.tar.gz: ff35d230ca178b9f5bc064e03b72aa9cd9adb88206e7ea30dbf99364ceb067b8
 SHA512:
-  metadata.gz: 243fbca142efbcca3f98933ec49fb1cb760f32dfa1c54fd1b37d5a809003bab9db296e7d5eafb8409e13617db72673579d89cd62d6abcf30d1bfefd36b4d755d
-  data.tar.gz: b37685ecda626db6426ffec6ede2139486cb34a3e83a9ef6e94b96a8609142fa9bd38f1b8b42265dce5387f992e2636583d3b1e87625f3aa4007e2157958b1ec
+  metadata.gz: 2f51bdbe47a354dfaf647fd852f06d8b650cedc5dd6416e4120f3160d075bd3e3a86debb2c5bc3ca14c4988ac06103b884d05070a13f48eefde5e65afdc99b9c
+  data.tar.gz: 79d52981b4f25213441b70c71acfcc1ccd6ebdba851a40d4576e5b2a1eca09b14ce5d1ec8e206aa48439392cb0a022a0e9ed158634bd6200a88e9cc4e1aebf41

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.11.10
+- Verbose error reason output (#469)
+
 ### 2.11.9
 - Use ConnectionRetryFailure in plugin specific retrying for consistency (#468)
 - Remove outdated generating hash_id_key code (#466)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.9'
+  s.version       = '2.11.10'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -73,7 +73,16 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         stats[:duplicates] += 1
       when 400 == status
         stats[:bad_argument] += 1
-        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
+        reason = ""
+        @plugin.log.on_debug do
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+            reason = " [error type]: #{item[write_operation]['error']['type']}"
+          end
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
+            reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
+          end
+        end
+        @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("400 - Rejected by Elasticsearch#{reason}"))
       else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -58,6 +58,47 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     JSON.parse(value)
   end
 
+  class TEST400ResponseReason < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::DEBUG}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::ElasticsearchErrorHandler.new(@plugin)
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(1, @plugin.error_events.size)
+      expected_log = "failed to parse"
+      exception_message = @plugin.error_events.first[:error].message
+      assert_true(exception_message.include?(expected_log),
+                  "Exception do not contain '#{exception_message}' '#{expected_log}'")
+      assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
+    end
+  end
+
   def test_dlq_400_responses
     records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.9
+  version: 2.11.10
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2018-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd