fluent-plugin-elasticsearch 1.15.2 → 1.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc1ee6d297a64d46320b2b48b8528faf9a377f1a9af9274c083f36e0d7f48c88
-  data.tar.gz: ff6de82c88a71a3f8a3c47951fadf0b555fe4118feb18fe911c4ef7b66178218
+  metadata.gz: 15e80a669ab0e24b83a279e1cbcc3240b76560b445319b7af7b2f719523b86e1
+  data.tar.gz: df01b4f813ffeba65619b778d418384b1cf3df3323929cf2921cf1273a336738
 SHA512:
-  metadata.gz: 3123a75aeee15020ec775457d36bfe6eaf52ba22309163c37859ca096556605dc400a98495f587f954e2e7b4d4ad2aa9cf9b7010c9c75158d911feaa6e181fa9
-  data.tar.gz: 506fc86e5930c18c38992d477d7865363bd351851e141879c67834dca05828f710795fc63725a6f5bc7fde9732660fa97a5caec45d6361b6c974e391fa1ef36c
+  metadata.gz: 2e44489b1f603ae61d9566c4cd0d6c16a7cec6ee5c701053f884ee284945fa2f48407674108223ec6c4c7127daabcaf17b6eb534b06d8c26339dd3aaba2fa908
+  data.tar.gz: cee4380e5d0aecfd9305bdbc68eaf3c2dbc9fef6b0e8f88f05ad4870a33d3b330983118a6ebe492ac4645805ec23c7acc270b91d37d6a222ed03ebc5ae28c6a5

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.16.0
+- evaluate bulk request failures and reroute failed messages (#405)
+
 ### 1.15.2
 - handle case where stats not processed in order; add testing (#410)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.15.2'
+  s.version       = '1.16.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -1,23 +1,41 @@
+require 'fluent/event'
 require_relative 'elasticsearch_constants'
 
 class Fluent::ElasticsearchErrorHandler
   include Fluent::ElasticsearchConstants
 
-  attr_accessor :records, :bulk_message_count
-  class BulkIndexQueueFull < StandardError; end
-  class ElasticsearchOutOfMemory < StandardError; end
+  attr_accessor :bulk_message_count
   class ElasticsearchVersionMismatch < StandardError; end
-  class UnrecognizedElasticsearchError < StandardError; end
   class ElasticsearchError < StandardError; end
-  def initialize(plugin, records = 0, bulk_message_count = 0)
+
+  def initialize(plugin)
     @plugin = plugin
-    @records = records
-    @bulk_message_count = bulk_message_count
   end
 
-  def handle_error(response)
+  def handle_error(response, tag, chunk, bulk_message_count)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array) 
+      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+        raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
     stats = Hash.new(0)
-    response['items'].each do |item|
+    meta = {}
+    header = {}
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        @plugin.process_message(tag, meta, header, time, processrecord, bulk_message)
+      rescue => e
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
       if item.has_key?(@plugin.write_operation)
         write_operation = @plugin.write_operation
       elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
@@ -41,13 +59,19 @@ class Fluent::ElasticsearchErrorHandler
         stats[:successes] += 1
       when CREATE_OP == write_operation && 409 == status
         stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        @plugin.router.emit_error_event(tag, time, rawrecord, '400 - Rejected by Elasticsearch')
       else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
+          stats[type] += 1
+          retry_stream.add(time, rawrecord)
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
           stats[:errors_bad_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, "#{status} - No error type provided in the response")
           next
         end
         stats[type] += 1
@@ -58,19 +82,6 @@ class Fluent::ElasticsearchErrorHandler
       stats.each_pair { |key, value| msg << "#{value} #{key}" }
       @plugin.log.debug msg.join(', ')
     end
-    case
-    when stats[:errors_bad_resp] > 0
-      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
-      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
-    when stats[:successes] + stats[:duplicates] == bulk_message_count
-      @plugin.log.info("retry succeeded - successes=#{stats[:successes]} duplicates=#{stats[:duplicates]}")
-    when stats['es_rejected_execution_exception'] > 0
-      raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
-    when stats['out_of_memory_error'] > 0
-      raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
-    else
-      @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
-      raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
-    end
+    raise Fluent::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
   end
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -10,6 +10,7 @@ rescue LoadError
 end
 
 require 'fluent/output'
+require 'fluent/event'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
@@ -17,6 +18,16 @@ require_relative 'elasticsearch_index_template'
 class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   class ConnectionFailure < StandardError; end
 
+  # RetryStreamError privides a stream to be
+  # put back in the pipeline for cases where a bulk request
+  # failed (e.g some records succeed while others failed)
+  class RetryStreamError < StandardError
+    attr_reader :retry_stream
+    def initialize(retry_stream)
+      @retry_stream = retry_stream
+    end
+  end
+
   Fluent::Plugin.register_output('elasticsearch', self)
 
   config_param :host, :string,  :default => 'localhost'
@@ -314,22 +325,21 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   end
 
   def write_objects(tag, chunk)
+    bulk_message_count = 0
     bulk_message = ''
     header = {}
     meta = {}
-    @error = Fluent::ElasticsearchErrorHandler.new(self)
-
     chunk.msgpack_each do |time, record|
-      @error.records += 1
       next unless record.is_a? Hash
       begin
         process_message(tag, meta, header, time, record, bulk_message)
+        bulk_message_count += 1
       rescue=>e
         router.emit_error_event(tag, time, record, e)
       end
     end
 
-    send_bulk(bulk_message) unless bulk_message.empty?
+    send_bulk(bulk_message, tag, chunk, bulk_message_count) unless bulk_message.empty?
     bulk_message.clear
   end
 
@@ -398,7 +408,6 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     end
 
     append_record_to_messages(@write_operation, meta, header, record, bulk_message)
-    @error.bulk_message_count += 1
   end
 
   # returns [parent, child_key] of child described by path array in record's tree
@@ -408,11 +417,18 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     [parent_object, path[-1]]
   end
 
-  def send_bulk(data)
+  # send_bulk given a specific bulk request, the original tag,
+  # chunk, and bulk_message_count
+  def send_bulk(data, tag, chunk, bulk_message_count)
     retries = 0
     begin
       response = client.bulk body: data
-      @error.handle_error(response) if response['errors']
+      if response['errors']
+        error = Fluent::ElasticsearchErrorHandler.new(self)
+        error.handle_error(response, tag, chunk, bulk_message_count) 
+      end
+    rescue RetryStreamError => e
+      router.emit_stream(tag, e.retry_stream)
     rescue *client.transport.host_unreachable_exceptions => e
       if retries < 2
         retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'
 
@@ -6,10 +7,35 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
-    attr_reader :write_operation
+    attr_reader :write_operation, :error_events
     def initialize(log)
       @log = log
       @write_operation = 'index'
+      @error_events = Fluent::MultiEventStream.new
+    end
+
+    def router
+      self
+    end
+
+    def emit_error_event(tag, time, record, e)
+       @error_events.add(time, record)
+    end
+
+    def process_message(tag, meta, header, time, record, bulk_message)
+      if record.has_key?('raise') && record['raise']
+        raise Exception('process_message')
+      end
+    end
+  end
+
+  class MockChunk
+    def initialize(records)
+      @records = records
+      @index = 0
+    end
+    def msgpack_each
+      @records.each { |item| yield(item[:time],item[:record]) }
     end
   end
 
@@ -31,7 +57,8 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     JSON.parse(value)
   end
 
-  def test_errors 
+  def test_dlq_400_responses
+    records = [{time: 123, record: {"foo" => "bar"}}]
     response = parse_response(%({
       "took" : 0,
       "errors" : true,
@@ -39,148 +66,44 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         {
           "create" : {
             "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
             "status" : 400,
-            "error" : {
-              "type" : "some unrecognized type",
+            "_type"  : "bar",
               "reason":"unrecognized error"
             }
-          }
         }
       ]
-    }))
-
-    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchError do 
-        @handler.handle_error(response)
-    end
-
+     }))
+    chunk = MockChunk.new(records)
+    @handler.handle_error(response, 'atag', chunk, records.length)
+    assert_equal(1, @plugin.error_events.instance_variable_get(:@time_array).size)
   end
 
-  def test_elasticsearch_version_mismatch_raises_error
-    response = parse_response(%(
-      {
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "reason":"some error to cause version mismatch"
-            }
-          }
-        }
-      ]
-      }
-    ))
-
-    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch do 
-        @handler.handle_error(response)
+  def test_retry_error
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
+    chunk = MockChunk.new(records)
 
-  end
-
-  def test_retry_with_successes_and_duplicates
-    response = parse_response(%(
-      {
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409,
-            "error" : {
-              "reason":"duplicate ID"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        }
-      ]
-      }
-    ))
-
-    @plugin.instance_variable_set(:@write_operation, 'create')
-    @handler.instance_variable_set(:@bulk_message_count, 2)
-    @handler.handle_error(response)
-    assert_match /retry succeeded - successes=1 duplicates=1/, @log.out.logs[0]
-  end
-
-  def test_bulk_rejection_errors
     response = parse_response(%({
-      "took" : 0,
+      "took" : 1,
       "errors" : true,
       "items" : [
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
+            "_id" : "1",
+            "status" : 201
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "2",
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
@@ -192,50 +115,39 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
+            "_id" : "3",
+            "status" : 409
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
+            "_id" : "5",
+            "status" : 500,
+            "error" : {
+              "reason":"unrecognized error - no type field"
+            }
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "6",
             "status" : 429,
             "error" : {
               "type" : "es_rejected_execution_exception",
-              "reason":"Elasticsearch could not process bulk index request"
+              "reason":"unable to fulfill request at this time, try again later"
             }
           }
-        }
-      ]
-    }))
-
-    assert_raise Fluent::ElasticsearchErrorHandler::BulkIndexQueueFull do
-        @handler.handle_error(response)
-    end
-
-  end
-
-  def test_out_of_memory_errors
-    response = parse_response(%({
-      "took" : 0,
-      "errors" : true,
-      "items" : [
+        },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
+            "_id" : "7",
+            "status" : 400,
             "error" : {
               "type" : "some unrecognized type",
               "reason":"unrecognized error"
@@ -246,48 +158,35 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "8",
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
               "reason":"unrecognized error"
             }
           }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 400,
-            "error" : {
-              "type" : "out_of_memory_error",
-              "reason":"Elasticsearch exhausted its heap"
-            }
-          }
         }
       ]
     }))
 
-    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchOutOfMemory do
-        @handler.handle_error(response)
+    begin
+      failed = false
+      @handler.handle_error(response, 'atag', chunk, response['items'].length)
+    rescue Fluent::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        e.retry_stream.each {|time, record| records << record}
+      end
+      assert_equal 3, records.length
+      assert_equal 2, records[0]['_id']
+      assert_equal 6, records[1]['_id']
+      assert_equal 8, records[2]['_id']
+      errors = @plugin.error_events.collect {|time, record| record}
+      assert_equal 2, errors.length
+      assert_equal 5, errors[0]['_id']
+      assert_equal 7, errors[1]['_id']
     end
+    assert_true failed
 
   end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -150,33 +150,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
   end
 
-  def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "type" => "some-other-type",
-      "reason" => "some-other-reason"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 504, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "category" => "some-other-type",
-      "reason" => "some-other-reason"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_index_to_create(url="http://localhost:9200/_bulk")
-    error = {
-      "category" => "some-other-type",
-      "reason" => "some-other-reason",
-      "type" => "some-other-type"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 0, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
   def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
     error = {
       "category" => "some-other-type",
@@ -1377,47 +1350,68 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_bulk_error
     stub_elastic_ping
-    stub_elastic_bulk_error
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchError) {
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.run
-    }
-  end
-
-  def test_bulk_error_version_mismatch
-    stub_elastic_ping
-    stub_elastic_version_mismatch
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.run
-    }
-  end
-
-  def test_bulk_index_into_a_create
-    stub_elastic_ping
-    stub_elastic_index_to_create
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchError) {
-      driver.emit(sample_record)
-      driver.run
-    }
-    assert(index_cmds[0].has_key?("create"))
-  end
-
-  def test_bulk_unexpected_response_op
-    stub_elastic_ping
-    stub_elastic_unexpected_response_op
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
-      driver.emit(sample_record)
-      driver.run
-    }
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 201
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "_id" : "abc",
+                "status" : 409
+              }
+            }
+           ]
+        })
+     }
+    end)
+
+    driver.emit(sample_record, 1)
+    driver.emit(sample_record, 2)
+    driver.emit(sample_record, 3)
+    driver.emit(sample_record, 4)
+
+    driver.expect_emit('test', 1, sample_record)
+    driver.expect_emit('test', 3, sample_record)
+    driver.run
   end
 
   def test_update_should_not_write_if_theres_no_id
@@ -1592,4 +1586,5 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
     assert(index_cmds[0].has_key?("create"))
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.15.2
+  version: 1.16.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd