fluent-plugin-elasticsearch 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba424a0c98f88f3fe3a85339d7e71b526de5dbba
-  data.tar.gz: ff250dcf41528631a3b39cc81628c8a4a7baccd7
+  metadata.gz: 16f9f3e664e714be9d1e7fee4b3f8a1683ec9728
+  data.tar.gz: beb9fa49cb58739036a3f2c1cda6ee936fbd873d
 SHA512:
-  metadata.gz: 0f12b278f5da2a6dcd37c5ac74dd16360753c67ea5fafd6810f501687fd9b96854e08556a87af849a8a1eb92cf14af176cd65346d9677c70d38e25d8f96467cc
-  data.tar.gz: 4773c01613026ec8f5ac727388ca033dbd9ab900922c5e00fb6709988cc186b67c22f297b26db22bd35ddbbda7f7605a4ba4f2865078d6c0fdc64e06a0338032
+  metadata.gz: 8527ebae2dbdb312cc6ed32af668077f9560b74db612a4f5d381f758e4e9cb4aa6230a6859a4a72c3f2b944b1fbf07292a49804cd4648a81392259938f133bab
+  data.tar.gz: 887830c553b5ed0350bf64aba4f38d24c28c557fad5493c3ae5f350049269737645e93addfac89b0c252898d07cc127707a9d35f23470ea45cd35f5df48839e0

data/History.md

@@ -4,6 +4,9 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.1.0
+- Retry on certain errors from Elasticsearch (#322)
+
 ### 2.0.1
 - Releasing generating hash id mechanism to avoid records duplication feature.
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.0.1'
+  s.version       = '2.1.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_constants.rb

@@ -0,0 +1,13 @@
+module Fluent
+  module Plugin
+    module ElasticsearchConstants
+      BODY_DELIMITER = "\n".freeze
+      UPDATE_OP = "update".freeze
+      UPSERT_OP = "upsert".freeze
+      CREATE_OP = "create".freeze
+      INDEX_OP = "index".freeze
+      ID_FIELD = "_id".freeze
+      TIMESTAMP_FIELD = "@timestamp".freeze
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -0,0 +1,96 @@
+require_relative 'elasticsearch_constants'
+
+class Fluent::Plugin::ElasticsearchErrorHandler
+  include Fluent::Plugin::ElasticsearchConstants
+
+  attr_accessor :records, :bulk_message_count
+  class BulkIndexQueueFull < StandardError; end
+  class ElasticsearchOutOfMemory < StandardError; end
+  class ElasticsearchVersionMismatch < StandardError; end
+  class UnrecognizedElasticsearchError < StandardError; end
+  class ElasticsearchError < StandardError; end
+  def initialize(plugin, records = 0, bulk_message_count = 0)
+    @plugin = plugin
+    @records = records
+    @bulk_message_count = bulk_message_count
+  end
+
+  def handle_error(response)
+    errors = Hash.new(0)
+    errors_bad_resp = 0
+    errors_unrecognized = 0
+    successes = 0
+    duplicates = 0
+    bad_arguments = 0
+    response['items'].each do |item|
+      if item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values (ES 2.x)
+        errors_bad_resp += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values (ES 2.x)
+        errors_bad_resp += 1
+        next
+      end
+      case
+      when CREATE_OP == write_operation && 409 == status
+        duplicates += 1
+      when 400 == status
+        bad_arguments += 1
+        @plugin.log.debug "Elasticsearch rejected document: #{item}"
+      when [429, 500].include?(status)
+        if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values (ES 2.x)
+          errors_bad_resp += 1
+          next
+        end
+        errors[type] += 1
+      when [200, 201].include?(status)
+        successes += 1
+      else
+        errors_unrecognized += 1
+      end
+    end
+    if errors_bad_resp > 0
+      msg = "Unable to parse error response from Elasticsearch, likely an API version mismatch  #{response}"
+      @plugin.log.error msg
+      raise ElasticsearchVersionMismatch, msg
+    end
+    if bad_arguments > 0
+      @plugin.log.warn "Elasticsearch rejected #{bad_arguments} documents due to invalid field arguments"
+    end
+    if duplicates > 0
+      @plugin.log.info "Encountered #{duplicates} duplicate(s) of #{successes} indexing chunk, ignoring"
+    end
+    msg = "Indexed (op = #{@plugin.write_operation}) #{successes} successfully, #{duplicates} duplicate(s), #{bad_arguments} bad argument(s), #{errors_unrecognized} unrecognized error(s)"
+    errors.each_key do |key|
+      msg << ", #{errors[key]} #{key} error(s)"
+    end
+    @plugin.log.debug msg
+    if errors_unrecognized > 0
+      raise UnrecognizedElasticsearchError, "Unrecognized elasticsearch errors returned, retrying  #{response}"
+    end
+    errors.each_key do |key|
+      case key
+      when 'out_of_memory_error'
+        raise ElasticsearchOutOfMemory, "Elasticsearch has exhausted its heap, retrying"
+      when 'es_rejected_execution_exception'
+        raise BulkIndexQueueFull, "Bulk index queue is full, retrying"
+      else
+        raise ElasticsearchError, "Elasticsearch errors returned, retrying  #{response}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -10,6 +10,8 @@ rescue LoadError
 end
 
 require 'fluent/plugin/output'
+require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
 require_relative 'generate_hash_id_support'
 
@@ -81,6 +83,7 @@ module Fluent::Plugin
 
     include Fluent::ElasticsearchIndexTemplate
     include Fluent::Plugin::GenerateHashIdSupport
+    include Fluent::Plugin::ElasticsearchConstants
 
     def initialize
       super
@@ -251,14 +254,6 @@ module Fluent::Plugin
       end.join(', ')
     end
 
-    BODY_DELIMITER = "\n".freeze
-    UPDATE_OP = "update".freeze
-    UPSERT_OP = "upsert".freeze
-    CREATE_OP = "create".freeze
-    INDEX_OP = "index".freeze
-    ID_FIELD = "_id".freeze
-    TIMESTAMP_FIELD = "@timestamp".freeze
-
     def append_record_to_messages(op, meta, header, record, msgs)
       case op
       when UPDATE_OP, UPSERT_OP
@@ -334,8 +329,10 @@ module Fluent::Plugin
 
       tag = chunk.metadata.tag
       logstash_prefix, index_name = expand_placeholders(chunk.metadata)
+      @error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
 
       chunk.msgpack_each do |time, record|
+        @error.records += 1
         next unless record.is_a? Hash
 
         if @flatten_hashes
@@ -402,6 +399,7 @@ module Fluent::Plugin
         end
 
         append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+        @error.bulk_message_count += 1
       end
 
       send_bulk(bulk_message) unless bulk_message.empty?
@@ -420,6 +418,7 @@ module Fluent::Plugin
       begin
         response = client.bulk body: data
         if response['errors']
+          @error.handle_error(response)
           log.error "Could not push log to Elasticsearch: #{response}"
         end
       rescue *client.transport.host_unreachable_exceptions => e

data/test/plugin/test_out_elasticsearch.rb

@@ -1,6 +1,7 @@
 require 'helper'
 require 'date'
 require 'fluent/test/helpers'
+require 'json'
 require 'fluent/test/driver/output'
 require 'flexmock/test_unit'
 
@@ -59,6 +60,130 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  def make_response_body(req, error_el = nil, error_status = nil, error = nil)
+    req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
+    items = []
+    count = 0
+    ids = 1
+    op = nil
+    index = nil
+    type = nil
+    id = nil
+    req_index_cmds.each do |cmd|
+      if count.even?
+        op = cmd.keys[0]
+        index = cmd[op]['_index']
+        type = cmd[op]['_type']
+        if cmd[op].has_key?('_id')
+          id = cmd[op]['_id']
+        else
+          # Note: this appears to be an undocumented feature of Elasticsearch
+          # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
+          # When you submit an "index" write_operation, with no "_id" field in the
+          # metadata header, Elasticsearch will turn this into a "create"
+          # operation in the response.
+          if "index" == op
+            op = "create"
+          end
+          id = ids
+          ids += 1
+        end
+      else
+        item = {
+          op => {
+            '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
+            '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
+            'status' => op == 'create' ? 201 : 200
+          }
+        }
+        items.push(item)
+      end
+      count += 1
+    end
+    if !error_el.nil? && !error_status.nil? && !error.nil?
+      op = items[error_el].keys[0]
+      items[error_el][op].delete('_version')
+      items[error_el][op].delete('_shards')
+      items[error_el][op]['error'] = error
+      items[error_el][op]['status'] = error_status
+      errors = true
+    else
+      errors = false
+    end
+    @index_cmds = items
+    body = { 'took' => 6, 'errors' => errors, 'items' => items }
+    return body.to_json
+  end
+
+  def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "mapper_parsing_exception",
+      "reason" => "failed to parse [...]",
+      "caused_by" => {
+        "type" => "illegal_argument_exception",
+        "reason" => "Invalid format: \"...\""
+      }
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "some-unrecognized-error",
+      "reason" => "some message printed here ...",
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "es_rejected_execution_exception",
+      "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "out_of_memory_error",
+      "reason" => "Java heap space"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
+    error = {
+      "type" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 504, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_index_to_create(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason",
+      "type" => "some-other-type"
+    }
+    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 0, 500, error), :headers => { 'Content-Type' => 'json' } } })
+  end
+
+  def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
+    error = {
+      "category" => "some-other-type",
+      "reason" => "some-other-reason"
+    }
+    stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
+  end
+
   def test_configure
     config = %{
       host     logs.google.com
@@ -1394,6 +1519,112 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
+  def test_bulk_bad_arguments
+    driver = driver('@log_level debug')
+
+    stub_elastic_ping
+    stub_elastic_bad_argument
+
+    driver.run(default_tag: 'test', shutdown: false) do
+      driver.feed(sample_record)
+      driver.feed(sample_record)
+      driver.feed(sample_record)
+    end
+
+    matches = driver.logs.grep /Elasticsearch rejected document:/
+    assert_equal(1, matches.length, "Message 'Elasticsearch rejected document: ...' was not emitted")
+    matches = driver.logs.grep /documents due to invalid field arguments/
+    assert_equal(1, matches.length, "Message 'Elasticsearch rejected # documents due to invalid field arguments ...' was not emitted")
+  end
+
+  def test_bulk_error
+    stub_elastic_ping
+    stub_elastic_bulk_error
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+      end
+    }
+  end
+
+  def test_bulk_error_version_mismatch
+    stub_elastic_ping
+    stub_elastic_version_mismatch
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+      end
+    }
+  end
+
+  def test_bulk_error_unrecognized_error
+    stub_elastic_ping
+    stub_elastic_unrecognized_error
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::UnrecognizedElasticsearchError) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+      end
+    }
+  end
+
+  def test_bulk_error_out_of_memory
+    stub_elastic_ping
+    stub_elastic_out_of_memory
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchOutOfMemory) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+      end
+    }
+  end
+
+  def test_bulk_error_queue_full
+    stub_elastic_ping
+    stub_elastic_bulk_rejected
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+        driver.feed(sample_record)
+      end
+    }
+  end
+
+  def test_bulk_index_into_a_create
+    stub_elastic_ping
+    stub_elastic_index_to_create
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
+    }
+    assert(index_cmds[0].has_key?("create"))
+  end
+
+  def test_bulk_unexpected_response_op
+    stub_elastic_ping
+    stub_elastic_unexpected_response_op
+
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
+    }
+  end
+
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic_ping

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - diogo
@@ -143,6 +143,8 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
+- lib/fluent/plugin/elasticsearch_constants.rb
+- lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/generate_hash_id_support.rb
 - lib/fluent/plugin/out_elasticsearch.rb