fluent-plugin-elasticsearch 2.9.2 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6294313d2bbda7a0933074c22b9d94db7d0a6ae7e7518210a0b56fa6a5ae2a2
-  data.tar.gz: a94f6a5ce9d4e79c3280d512cb8acd9174aac263179f09f18088f50f6a1f9a62
+  metadata.gz: f778421b8290ea1db16ad57f1c0be9a35e656d201e5b52c502ae9301c60a67c3
+  data.tar.gz: 7e72d88f792ac7a959faa393e91fcc08917ada672860730352219b5d26b167b4
 SHA512:
-  metadata.gz: d0184d1b690d6c9118bcb419e6904411b61dd8cac289f59e78ba554fbe3c953ac7b697b5271a491fbcf5b391d5691a7c3982288ed10bf6b9e1eca54a705450cb
-  data.tar.gz: e0efbbd71009eaa75e8372677fbd169a54f1d143b2ce12df112c075af8b523a18650472ea6c4692267ca39cb48f30204d81dbd556bc18187d2c31a7c515ee4a8
+  metadata.gz: 8222728bf41a377029aa620f94f7d773529d4dbae47081158022c49b8ebb08aed224d93208db1d4933663ea5226decd1e99096c9890a0f59a58d22e8aca9d5ae
+  data.tar.gz: fe857ad374b96cfc4eeedb02642fc1bebfb4da43f0cba397789db9f74a6df6704ffb1c57b65524b00866e4b540c97130c387f6252bbf35fa1215d95c7f3c1f1a

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.10.0
+- Uplift Merge pull request #405 from jcantrill/sanitize_bulk (#414)
+
 ### 2.9.2
 - Uplift Merge pull request #410 from richm/v0.12-consistent-errors-and-tests (#411)
 - Add correct link for include_timestamp (#408)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.9.2'
+  s.version       = '2.10.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -1,23 +1,41 @@
+require 'fluent/event'
 require_relative 'elasticsearch_constants'
 
 class Fluent::Plugin::ElasticsearchErrorHandler
   include Fluent::Plugin::ElasticsearchConstants
 
-  attr_accessor :records, :bulk_message_count
-  class BulkIndexQueueFull < StandardError; end
-  class ElasticsearchOutOfMemory < StandardError; end
+  attr_accessor :bulk_message_count
   class ElasticsearchVersionMismatch < StandardError; end
-  class UnrecognizedElasticsearchError < StandardError; end
   class ElasticsearchError < StandardError; end
-  def initialize(plugin, records = 0, bulk_message_count = 0)
+
+  def initialize(plugin)
     @plugin = plugin
-    @records = records
-    @bulk_message_count = bulk_message_count
   end
 
-  def handle_error(response)
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array)
+      raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+        raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
     stats = Hash.new(0)
-    response['items'].each do |item|
+    meta = {}
+    header = {}
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
+      rescue => e
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
       if item.has_key?(@plugin.write_operation)
         write_operation = @plugin.write_operation
       elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
@@ -41,13 +59,19 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         stats[:successes] += 1
       when CREATE_OP == write_operation && 409 == status
         stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        @plugin.router.emit_error_event(tag, time, rawrecord, '400 - Rejected by Elasticsearch')
       else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
+          stats[type] += 1
+          retry_stream.add(time, rawrecord)
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
           stats[:errors_bad_resp] += 1
+          @plugin.router.emit_error_event(tag, time, rawrecord, "#{status} - No error type provided in the response")
           next
         end
         stats[type] += 1
@@ -58,19 +82,6 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       stats.each_pair { |key, value| msg << "#{value} #{key}" }
       @plugin.log.debug msg.join(', ')
     end
-    case
-    when stats[:errors_bad_resp] > 0
-      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
-      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
-    when stats[:successes] + stats[:duplicates] == bulk_message_count
-      @plugin.log.info("retry succeeded - successes=#{stats[:successes]} duplicates=#{stats[:duplicates]}")
-    when stats['es_rejected_execution_exception'] > 0
-      raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
-    when stats['out_of_memory_error'] > 0
-      raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
-    else
-      @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
-      raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
-    end
+    raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
   end
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -10,6 +10,7 @@ rescue LoadError
 end
 
 require 'fluent/plugin/output'
+require 'fluent/event'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
@@ -18,6 +19,16 @@ module Fluent::Plugin
   class ElasticsearchOutput < Output
     class ConnectionFailure < StandardError; end
 
+    # RetryStreamError privides a stream to be
+    # put back in the pipeline for cases where a bulk request
+    # failed (e.g some records succeed while others failed)
+    class RetryStreamError < StandardError
+      attr_reader :retry_stream
+      def initialize(retry_stream)
+        @retry_stream = retry_stream
+      end
+    end
+
     helpers :event_emitter, :compat_parameters, :record_accessor
 
     Fluent::Plugin.register_output('elasticsearch', self)
@@ -375,26 +386,26 @@ EOC
     end
 
     def write(chunk)
+      bulk_message_count = 0
       bulk_message = ''
       header = {}
       meta = {}
 
       tag = chunk.metadata.tag
       extracted_values = expand_placeholders(chunk.metadata)
-      @error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
       @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
 
       chunk.msgpack_each do |time, record|
-        @error.records += 1
         next unless record.is_a? Hash
-
         begin
           process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+          bulk_message_count += 1
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end
       end
-      send_bulk(bulk_message) unless bulk_message.empty?
+
+      send_bulk(bulk_message, tag, chunk, bulk_message_count, extracted_values) unless bulk_message.empty?
       bulk_message.clear
     end
 
@@ -479,7 +490,6 @@ EOC
       end
 
       append_record_to_messages(@write_operation, meta, header, record, bulk_message)
-      @error.bulk_message_count += 1
     end
 
     # returns [parent, child_key] of child described by path array in record's tree
@@ -489,11 +499,18 @@ EOC
       [parent_object, path[-1]]
     end
 
-    def send_bulk(data)
+    # send_bulk given a specific bulk request, the original tag,
+    # chunk, and bulk_message_count
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values)
       retries = 0
       begin
         response = client.bulk body: data
-        @error.handle_error(response) if response['errors']
+        if response['errors']
+          error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
+          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+        end
+      rescue RetryStreamError => e
+        router.emit_stream(tag, e.retry_stream)
       rescue *client.transport.host_unreachable_exceptions => e
         if retries < 2
           retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'
 
@@ -6,10 +7,35 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
-    attr_reader :write_operation
+    attr_reader :write_operation, :error_events
     def initialize(log)
       @log = log
       @write_operation = 'index'
+      @error_events = Fluent::MultiEventStream.new
+    end
+
+    def router
+      self
+    end
+
+    def emit_error_event(tag, time, record, e)
+       @error_events.add(time, record)
+    end
+
+    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+      if record.has_key?('raise') && record['raise']
+        raise Exception('process_message')
+      end
+    end
+  end
+
+  class MockChunk
+    def initialize(records)
+      @records = records
+      @index = 0
+    end
+    def msgpack_each
+      @records.each { |item| yield(item[:time],item[:record]) }
     end
   end
 
@@ -27,7 +53,8 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     JSON.parse(value)
   end
 
-  def test_errors
+  def test_dlq_400_responses
+    records = [{time: 123, record: {"foo" => "bar"}}]
     response = parse_response(%({
       "took" : 0,
       "errors" : true,
@@ -35,148 +62,45 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         {
           "create" : {
             "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
             "status" : 400,
-            "error" : {
-              "type" : "some unrecognized type",
+            "_type"  : "bar",
               "reason":"unrecognized error"
             }
-          }
         }
       ]
-    }))
-
-    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError do
-        @handler.handle_error(response)
-    end
-
+     }))
+    chunk = MockChunk.new(records)
+    dummy_extracted_values = []
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    assert_equal(1, @plugin.error_events.instance_variable_get(:@time_array).size)
   end
 
-  def test_elasticsearch_version_mismatch_raises_error
-    response = parse_response(%(
-      {
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "reason":"some error to cause version mismatch"
-            }
-          }
-        }
-      ]
-      }
-    ))
-
-    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch do
-        @handler.handle_error(response)
+  def test_retry_error
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
+    chunk = MockChunk.new(records)
 
-  end
-
-  def test_retry_with_successes_and_duplicates
-    response = parse_response(%(
-      {
-      "took" : 0,
-      "errors" : true,
-      "items" : [
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409,
-            "error" : {
-              "reason":"duplicate ID"
-            }
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        }
-      ]
-      }
-    ))
-
-    @plugin.instance_variable_set(:@write_operation, 'create')
-    @handler.instance_variable_set(:@bulk_message_count, 2)
-    @handler.handle_error(response)
-    assert_match /retry succeeded - successes=1 duplicates=1/, @log.out.logs[0]
-  end
-
-  def test_bulk_rejection_errors
     response = parse_response(%({
-      "took" : 0,
+      "took" : 1,
       "errors" : true,
       "items" : [
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
-            "error" : {
-              "type" : "some unrecognized type",
-              "reason":"unrecognized error"
-            }
+            "_id" : "1",
+            "status" : 201
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "2",
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
@@ -188,50 +112,39 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
+            "_id" : "3",
+            "status" : 409
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
+            "_id" : "5",
+            "status" : 500,
+            "error" : {
+              "reason":"unrecognized error - no type field"
+            }
           }
         },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "6",
             "status" : 429,
             "error" : {
               "type" : "es_rejected_execution_exception",
-              "reason":"Elasticsearch could not process bulk index request"
+              "reason":"unable to fulfill request at this time, try again later"
             }
           }
-        }
-      ]
-    }))
-
-    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull do
-        @handler.handle_error(response)
-    end
-
-  end
-
-  def test_out_of_memory_errors
-    response = parse_response(%({
-      "took" : 0,
-      "errors" : true,
-      "items" : [
+        },
         {
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 500,
+            "_id" : "7",
+            "status" : 400,
             "error" : {
               "type" : "some unrecognized type",
               "reason":"unrecognized error"
@@ -242,48 +155,36 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           "create" : {
             "_index" : "foo",
             "_type"  : "bar",
-            "_id" : "abc",
+            "_id" : "8",
             "status" : 500,
             "error" : {
               "type" : "some unrecognized type",
               "reason":"unrecognized error"
             }
           }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 201
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 409
-          }
-        },
-        {
-          "create" : {
-            "_index" : "foo",
-            "_type"  : "bar",
-            "_id" : "abc",
-            "status" : 400,
-            "error" : {
-              "type" : "out_of_memory_error",
-              "reason":"Elasticsearch exhausted its heap"
-            }
-          }
         }
       ]
     }))
 
-    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchOutOfMemory do
-        @handler.handle_error(response)
+    begin
+      failed = false
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+    rescue Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        e.retry_stream.each {|time, record| records << record}
+      end
+      assert_equal 3, records.length
+      assert_equal 2, records[0]['_id']
+      assert_equal 6, records[1]['_id']
+      assert_equal 8, records[2]['_id']
+      errors = @plugin.error_events.collect {|time, record| record}
+      assert_equal 2, errors.length
+      assert_equal 5, errors[0]['_id']
+      assert_equal 7, errors[1]['_id']
     end
+    assert_true failed
 
   end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -170,33 +170,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
   end
 
-  def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "type" => "some-other-type",
-      "reason" => "some-other-reason"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 504, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "category" => "some-other-type",
-      "reason" => "some-other-reason"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_index_to_create(url="http://localhost:9200/_bulk")
-    error = {
-      "category" => "some-other-type",
-      "reason" => "some-other-reason",
-      "type" => "some-other-type"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 0, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
   def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
     error = {
       "category" => "some-other-type",
@@ -1748,51 +1721,70 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_bulk_error
     stub_elastic_ping
-    stub_elastic_bulk_error
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-      end
-    }
-  end
-
-  def test_bulk_error_version_mismatch
-    stub_elastic_ping
-    stub_elastic_version_mismatch
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-      end
-    }
-  end
-
-  def test_bulk_index_into_a_create
-    stub_elastic_ping
-    stub_elastic_index_to_create
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-      end
-    }
-    assert(index_cmds[0].has_key?("create"))
-  end
-
-  def test_bulk_unexpected_response_op
-    stub_elastic_ping
-    stub_elastic_unexpected_response_op
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-      end
-    }
+    stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+      { :status => 200,
+        :headers => { 'Content-Type' => 'json' },
+        :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 201
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "status" : 500,
+                "error" : {
+                  "type" : "some unrecognized type",
+                  "reason":"some error to cause version mismatch"
+                }
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc",
+                "_id" : "abc",
+                "status" : 409
+              }
+            }
+           ]
+        })
+     }
+    end)
+
+    driver.run(default_tag: 'test') do
+      driver.feed(1, sample_record)
+      driver.feed(2, sample_record)
+      driver.feed(3, sample_record)
+      driver.feed(4, sample_record)
+    end
+
+    expect = [['test', 1, sample_record],
+              ['test', 3, sample_record]]
+    assert_equal expect, driver.events
   end
 
   def test_update_should_not_write_if_theres_no_id
@@ -1979,4 +1971,5 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
     assert(index_cmds[0].has_key?("create"))
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.9.2
+  version: 2.10.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd