fluent-plugin-elasticsearch 1.10.3 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ee62e58133f9ebcd78a6ebfa0eb8adb741ea207
-  data.tar.gz: 848f36e2b35efe4013195a5b79eea21515973962
+  metadata.gz: 6d8cd27ced7ebf6f0c8d1a2026032b683af8eadd
+  data.tar.gz: 98591a5682b51f4ca97fb206620cec059c57ca74
 SHA512:
-  metadata.gz: 5f5f61edcc15c83d546411455603554fc2e2ff7d72cc88b1ccdebed4d2b411ce5cfb5625552ec4536911ecfe9ad02660b3a1634e38dd424f67dd55cb0f75be7a
-  data.tar.gz: 1508c853846a2142508e06a4e6c213377ede90cf5ae0c691ced885477787eb0e1df1a26c6338fe33d21e77e31dfc8c3ee740317b05e90af45bc7f6e43dae7459
+  metadata.gz: c9291c5f41c33732d5dc467e6171446670c1deca8ec79d2c01085e1aeda17bec50206573cede541766a0a6ddce2a566c7d8cc5d2531a410173b2ea07ad98c11e
+  data.tar.gz: c7c9c6badfddfd12190837ad504355d2758195c7318fef04b21ce67a0c6a051157a4b89aed074559f9f2696f7af9fc408d2f27ab07db0ae8c6e63214beee64bc

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 1.11.0
+- backport adding bulk errors handling (#324)
+
 ### 1.10.3
 - releasing generating hash id mechanism to avoid records duplication backport (#323)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.10.3'
+  s.version       = '1.11.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -0,0 +1,11 @@
+module Fluent
+  module ElasticsearchConstants
+    BODY_DELIMITER = "\n".freeze
+    UPDATE_OP = "update".freeze
+    UPSERT_OP = "upsert".freeze
+    CREATE_OP = "create".freeze
+    INDEX_OP = "index".freeze
+    ID_FIELD = "_id".freeze
+    TIMESTAMP_FIELD = "@timestamp".freeze
+  end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -0,0 +1,96 @@
+require_relative 'elasticsearch_constants'
+
+class Fluent::ElasticsearchErrorHandler
+  include Fluent::ElasticsearchConstants
+
+  attr_accessor :records, :bulk_message_count
+  class BulkIndexQueueFull < StandardError; end
+  class ElasticsearchOutOfMemory < StandardError; end
+  class ElasticsearchVersionMismatch < StandardError; end
+  class UnrecognizedElasticsearchError < StandardError; end
+  class ElasticsearchError < StandardError; end
+  def initialize(plugin, records = 0, bulk_message_count = 0)
+    @plugin = plugin
+    @records = records
+    @bulk_message_count = bulk_message_count
+  end
+
+  def handle_error(response)
+    errors = Hash.new(0)
+    errors_bad_resp = 0
+    errors_unrecognized = 0
+    successes = 0
+    duplicates = 0
+    bad_arguments = 0
+    response['items'].each do |item|
+      if item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values (ES 2.x)
+        errors_bad_resp += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values (ES 2.x)
+        errors_bad_resp += 1
+        next
+      end
+      case
+      when CREATE_OP == write_operation && 409 == status
+        duplicates += 1
+      when 400 == status
+        bad_arguments += 1
+        @plugin.log.debug "Elasticsearch rejected document: #{item}"
+      when [429, 500].include?(status)
+        if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values (ES 2.x)
+          errors_bad_resp += 1
+          next
+        end
+        errors[type] += 1
+      when [200, 201].include?(status)
+        successes += 1
+      else
+        errors_unrecognized += 1
+      end
+    end
+    if errors_bad_resp > 0
+      msg = "Unable to parse error response from Elasticsearch, likely an API version mismatch  #{response}"
+      @plugin.log.error msg
+      raise ElasticsearchVersionMismatch, msg
+    end
+    if bad_arguments > 0
+      @plugin.log.warn "Elasticsearch rejected #{bad_arguments} documents due to invalid field arguments"
+    end
+    if duplicates > 0
+      @plugin.log.info "Encountered #{duplicates} duplicate(s) of #{successes} indexing chunk, ignoring"
+    end
+    msg = "Indexed (op = #{@plugin.write_operation}) #{successes} successfully, #{duplicates} duplicate(s), #{bad_arguments} bad argument(s), #{errors_unrecognized} unrecognized error(s)"
+    errors.each_key do |key|
+      msg << ", #{errors[key]} #{key} error(s)"
+    end
+    @plugin.log.debug msg
+    if errors_unrecognized > 0
+      raise UnrecognizedElasticsearchError, "Unrecognized elasticsearch errors returned, retrying  #{response}"
+    end
+    errors.each_key do |key|
+      case key
+      when 'out_of_memory_error'
+        raise ElasticsearchOutOfMemory, "Elasticsearch has exhausted its heap, retrying"
+      when 'es_rejected_execution_exception'
+        raise BulkIndexQueueFull, "Bulk index queue is full, retrying"
+      else
+        raise ElasticsearchError, "Elasticsearch errors returned, retrying  #{response}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -10,6 +10,8 @@ rescue LoadError
 end
 
 require 'fluent/output'
+require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
 require_relative 'generate_hash_id_support'
 
@@ -69,6 +71,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
 
   include Fluent::ElasticsearchIndexTemplate
   include Fluent::GenerateHashIdSupport
+  include Fluent::ElasticsearchConstants
 
   def initialize
     super
@@ -305,8 +308,10 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     bulk_message = ''
     header = {}
     meta = {}
+    @error = Fluent::ElasticsearchErrorHandler.new(self)
 
     chunk.msgpack_each do |time, record|
+      @error.records += 1
       next unless record.is_a? Hash
 
       if @flatten_hashes
@@ -373,6 +378,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
       end
 
       append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+      @error.bulk_message_count += 1
     end
 
     send_bulk(bulk_message) unless bulk_message.empty?
@@ -391,6 +397,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     begin
       response = client.bulk body: data
       if response['errors']
+        @error.handle_error(response)
         log.error "Could not push log to Elasticsearch: #{response}"
       end
     rescue *client.transport.host_unreachable_exceptions => e

data/test/plugin/test_out_elasticsearch.rb

@@ -1,5 +1,6 @@
 require 'helper'
 require 'date'
+require 'json'
 require 'flexmock/test_unit'
 
 class ElasticsearchOutput < Test::Unit::TestCase
@@ -56,6 +57,130 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  def make_response_body(req, error_el = nil, error_status = nil, error = nil)
+    req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
+    items = []
+    count = 0
+    ids = 1
+    op = nil
+    index = nil
+    type = nil
+    id = nil
+    req_index_cmds.each do |cmd|
+      if count.even?
+        op = cmd.keys[0]
+        index = cmd[op]['_index']
+        type = cmd[op]['_type']
+        if cmd[op].has_key?('_id')
+          id = cmd[op]['_id']
+        else
+          # Note: this appears to be an undocumented feature of Elasticsearch
+          # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
+          # When you submit an "index" write_operation, with no "_id" field in the
+          # metadata header, Elasticsearch will turn this into a "create"
+          # operation in the response.
+          if "index" == op
+            op = "create"
+          end
+          id = ids
+          ids += 1
+        end
+      else
+        item = {
+          op => {
+            '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
+            '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
+            'status' => op == 'create' ? 201 : 200
+          }
+        }
+        items.push(item)
+      end
+      count += 1
+    end
+    if !error_el.nil? && !error_status.nil? && !error.nil?
+      op = items[error_el].keys[0]
+      items[error_el][op].delete('_version')
+      items[error_el][op].delete('_shards')
+      items[error_el][op]['error'] = error
+      items[error_el][op]['status'] = error_status
+      errors = true
+    else
+      errors = false
+    end
+    @index_cmds = items
+    body = { 'took' => 6, 'errors' => errors, 'items' => items }
+    return body.to_json
+  end
+
+  def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "mapper_parsing_exception",
+      "reason" => "failed to parse [...]",
+      "caused_by" => {
+        "type" => "illegal_argument_exception",
+        "reason" => "Invalid format: \"...\""
+      }
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "some-unrecognized-error",
+      "reason" => "some message printed here ...",
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "es_rejected_execution_exception",
+      "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "out_of_memory_error",
+      "reason" => "Java heap space"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 504, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_index_to_create(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason",
+      "type" => "some-other-type"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 0, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
+  end
+
   def test_configure
     config = %{
       host     logs.google.com
@@ -1181,6 +1306,106 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
+  def test_bulk_bad_arguments
+    log = driver.instance.router.emit_error_handler.log
+    log.level = 'debug'
+    driver = driver('@log_level debug')
+
+    stub_elastic_ping
+    stub_elastic_bad_argument
+
+    driver.emit(sample_record)
+    driver.emit(sample_record)
+    driver.emit(sample_record)
+    driver.run
+
+    matches = log.out.logs.grep /Elasticsearch rejected document:/
+    assert_equal(1, matches.length, "Message 'Elasticsearch rejected document: ...' was not emitted")
+    matches = log.out.logs.grep /documents due to invalid field arguments/
+    assert_equal(1, matches.length, "Message 'Elasticsearch rejected # documents due to invalid field arguments ...' was not emitted")
+  end
+
+  def test_bulk_error
+    stub_elastic_ping
+    stub_elastic_bulk_error
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchError) {
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
+  def test_bulk_error_version_mismatch
+    stub_elastic_ping
+    stub_elastic_version_mismatch
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
+  def test_bulk_error_unrecognized_error
+    stub_elastic_ping
+    stub_elastic_unrecognized_error
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::UnrecognizedElasticsearchError) {
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
+  def test_bulk_error_out_of_memory
+    stub_elastic_ping
+    stub_elastic_out_of_memory
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchOutOfMemory) {
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
+  def test_bulk_error_queue_full
+    stub_elastic_ping
+    stub_elastic_bulk_rejected
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::BulkIndexQueueFull) {
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
+  def test_bulk_index_into_a_create
+    stub_elastic_ping
+    stub_elastic_index_to_create
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchError) {
+      driver.emit(sample_record)
+      driver.run
+    }
+    assert(index_cmds[0].has_key?("create"))
+  end
+
+  def test_bulk_unexpected_response_op
+    stub_elastic_ping
+    stub_elastic_unexpected_response_op
+
+    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
+      driver.emit(sample_record)
+      driver.run
+    }
+  end
+
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic_ping

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.11.0
 platform: ruby
 authors:
 - diogo
@@ -144,6 +144,8 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
+- lib/fluent/plugin/elasticsearch_constants.rb
+- lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb