fluent-plugin-elasticsearch 1.14.0 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 995e9b0bb1ae8dc5fbf60b80c6d038c0b1f5b696aa4e27842ab23b1609eeef74
-  data.tar.gz: c1b249d0d02c7381b0a115fccad53e68ea170906101913c36a2b52c294bb3275
+  metadata.gz: 709ac4334da5ddeae76a8cd290cb54b3d972458515e8e2855c1c56f216a078ca
+  data.tar.gz: 17cb0aae657c8d2bafea64e95432483667ffa3e2d1ae531a0dd34880ce580347
 SHA512:
-  metadata.gz: 158848be63d3b28213856a45026b65e769e480285b43400a5600a7f5e7d01aed24f44bbbc9fad60d0530f64dad2717cc90d57c0ac84f75c1f46c3f439ab56a5f
-  data.tar.gz: 223f97c244f7488f364d3a553e44b0d1fddf4c3f9c93545dd696e84e6a2bfd92abc8565e42ce7b5c402e4ca9bca242b6ac66a88337165579b6b818a7d3261ff2
+  metadata.gz: 157d3d0d15eef38c8632c4d1189bfa651ad2902300ce0f20cb56ad1a75bbb0ac6f7546a3ec93e8769d14d9feaf774cab075f049c97218de01c2bd52e89fb7ff5
+  data.tar.gz: 11cb945b1898b87921ea642a873b47352080d667d864965c933481c6601b45cbdac1246ce9e9d996e2a4a5d18f46bdb462a664ab001f8337a801c329aa627f36

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 1.15.0
+- revert dlq to use router.emit_error_event instead (#402)
+- Don't log full response on error (#399)
+
 ### 1.14.0
 - introduce dead letter queue to handle issues unpacking file buffer chunks (#398)
 

data/README.md

@@ -49,7 +49,6 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [time_parse_error_tag](#time_parse_error_tag)
   + [reconnect_on_error](#reconnect_on_error)
   + [with_transporter_log](#with_transporter_log)
-  + [dlq_handler](#dlq_handler)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
@@ -475,17 +474,6 @@ We recommend to set this true if you start to debug this plugin.
 with_transporter_log true
 ```
 
-### dlq_handler
-Adds an error handler for processing corrupt messages from message buffers.
-There are [known cases](https://bugzilla.redhat.com/show_bug.cgi?id=1562004) where
-fluentd is stuck processing messages because the file buffer is corrupt.  Fluentd
-is unable to clear faulty buffer chunks.
-
-```
-dlq_handler {'type':'drop'}        #default is to log and drop messages
-dlq_handler {'type':'file', 'dir':'/tmp/fluentd/dlq', 'max_files':5, 'max_file_size':104857600}
-```
-
 ### Client/host certificate options
 
 Need to verify Elasticsearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.14.0'
+  s.version       = '1.15.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -16,12 +16,7 @@ class Fluent::ElasticsearchErrorHandler
   end
 
   def handle_error(response)
-    errors = Hash.new(0)
-    errors_bad_resp = 0
-    errors_unrecognized = 0
-    successes = 0
-    duplicates = 0
-    bad_arguments = 0
+    stats = Hash.new(0)
     response['items'].each do |item|
       if item.has_key?(@plugin.write_operation)
         write_operation = @plugin.write_operation
@@ -30,7 +25,7 @@ class Fluent::ElasticsearchErrorHandler
       else
         # When we don't have an expected ops field, something changed in the API
         # expected return values (ES 2.x)
-        errors_bad_resp += 1
+        stats[:errors_bad_resp] += 1
         next
       end
       if item[write_operation].has_key?('status')
@@ -38,58 +33,44 @@ class Fluent::ElasticsearchErrorHandler
       else
         # When we don't have a status field, something changed in the API
         # expected return values (ES 2.x)
-        errors_bad_resp += 1
+        stats[:errors_bad_resp] += 1
         next
       end
       case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
       when CREATE_OP == write_operation && 409 == status
-        duplicates += 1
-      when 400 == status
-        bad_arguments += 1
-        @plugin.log.debug "Elasticsearch rejected document: #{item}"
-      when [429, 500].include?(status)
+        stats[:duplicates] += 1
+      else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
-          errors_bad_resp += 1
+          stats[:errors_bad_resp] += 1
           next
         end
-        errors[type] += 1
-      when [200, 201].include?(status)
-        successes += 1
-      else
-        errors_unrecognized += 1
+        stats[type] += 1
       end
     end
-    if errors_bad_resp > 0
-      msg = "Unable to parse error response from Elasticsearch, likely an API version mismatch  #{response}"
-      @plugin.log.error msg
-      raise ElasticsearchVersionMismatch, msg
-    end
-    if bad_arguments > 0
-      @plugin.log.warn "Elasticsearch rejected #{bad_arguments} documents due to invalid field arguments"
-    end
-    if duplicates > 0
-      @plugin.log.info "Encountered #{duplicates} duplicate(s) of #{successes} indexing chunk, ignoring"
-    end
-    msg = "Indexed (op = #{@plugin.write_operation}) #{successes} successfully, #{duplicates} duplicate(s), #{bad_arguments} bad argument(s), #{errors_unrecognized} unrecognized error(s)"
-    errors.each_key do |key|
-      msg << ", #{errors[key]} #{key} error(s)"
+    if stats[:errors_bad_resp] > 0
+      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
+      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
     end
-    @plugin.log.debug msg
-    if errors_unrecognized > 0
-      raise UnrecognizedElasticsearchError, "Unrecognized elasticsearch errors returned, retrying  #{response}"
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
     end
-    errors.each_key do |key|
+    stats.each_key do |key|
       case key
       when 'out_of_memory_error'
-        raise ElasticsearchOutOfMemory, "Elasticsearch has exhausted its heap, retrying"
+        raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
       when 'es_rejected_execution_exception'
-        raise BulkIndexQueueFull, "Bulk index queue is full, retrying"
+        raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
       else
-        raise ElasticsearchError, "Elasticsearch errors returned, retrying  #{response}"
+        @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
+        raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
       end
     end
   end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -69,7 +69,6 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :reconnect_on_error, :bool, :default => false
   config_param :pipeline, :string, :default => nil
   config_param :with_transporter_log, :bool, :default => false
-  config_param :dlq_handler, :hash, :default => { 'type' =>'drop' }
 
   include Fluent::ElasticsearchIndexTemplate
   include Fluent::ElasticsearchConstants
@@ -131,43 +130,6 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
       log.warn "Consider to specify log_level with @log_level." unless log_level
     end
 
-    configure_dlq_handler
-
-  end
-
-  def configure_dlq_handler
-    dlq_type = @dlq_handler && @dlq_handler.is_a?(Hash) ? dlq_type = @dlq_handler['type'] : nil
-    return unless dlq_type
-
-    case dlq_type.downcase
-    when 'drop'
-      log.info('Configuring the DROP dead letter queue handler')
-      require_relative 'dead_letter_queue_drop_handler'
-      extend Fluent::DeadLetterQueueDropHandler
-    when 'file'
-      log.info("Configuring the File dead letter queue handler: ")
-      dir = @dlq_handler ['dir'] || '/var/lib/fluentd/dlq'
-      shift_age = @dlq_handler['max_files'] || 0
-      shift_size = @dlq_handler['max_file_size'] || 1048576
-      log.info("Configuring the File dead letter queue handler: ")
-      log.info("                Directory: #{dir}")
-      log.info("  Max number of DLQ files: #{shift_age}")
-      log.info("            Max file size: #{shift_size}")
-      unless Dir.exists?(dir)
-        Dir.mkdir(dir)
-        log.info("Created DLQ directory: '#{dir}'")
-      end
-      require 'logger'
-      require 'json'
-      file = File.join(dir, 'dlq')
-      @dlq_file = Logger.new(file, shift_age, shift_size)
-      @dlq_file.level = Logger::INFO
-      @dlq_file.formatter = proc { |severity, datetime, progname, msg| "#{msg.dump}\n" }
-      log.info ("Created DLQ file #{file}")
-
-      require_relative 'dead_letter_queue_file_handler'
-      extend Fluent::DeadLetterQueueFileHandler
-    end
   end
 
   def create_meta_config_map
@@ -363,7 +325,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
       begin
         process_message(tag, meta, header, time, record, bulk_message)
       rescue=>e
-        handle_chunk_error(self, tag, e, time, record)
+        router.emit_error_event(tag, time, record, e)
       end
     end
 
@@ -450,10 +412,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     retries = 0
     begin
       response = client.bulk body: data
-      if response['errors']
-        @error.handle_error(response)
-        log.error "Could not push log to Elasticsearch: #{response}"
-      end
+      @error.handle_error(response) if response['errors']
     rescue *client.transport.host_unreachable_exceptions => e
       if retries < 2
         retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -0,0 +1,122 @@
+require 'helper'
+require 'fluent/plugin/elasticsearch_error_handler'
+require 'json'
+
+class TestElasticsearchErrorHandler < Test::Unit::TestCase
+
+  class TestPlugin
+    attr_reader :log
+    def initialize(log)
+      @log = log
+    end
+
+    def write_operation
+      'index'
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+    @log = Fluent::Engine.log
+    plugin = TestPlugin.new(@log)
+    @handler =  Fluent::ElasticsearchErrorHandler.new(plugin)
+  end
+
+  def parse_response(value)
+    JSON.parse(value)
+  end
+
+  def test_errors 
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 400,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchError do 
+        @handler.handle_error(response)
+    end
+
+  end
+
+  def test_elasticsearch_version_mismatch_raises_error
+    response = parse_response(%(
+      {
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        }
+      ]
+      }
+    ))
+
+    assert_raise Fluent::ElasticsearchErrorHandler::ElasticsearchVersionMismatch do 
+        @handler.handle_error(response)
+    end
+
+  end
+
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -134,6 +134,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "es_rejected_execution_exception",
       "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
     }
@@ -142,6 +143,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "out_of_memory_error",
       "reason" => "Java heap space"
     }
@@ -150,6 +152,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "some-other-type",
       "reason" => "some-other-reason"
     }
@@ -158,6 +161,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "category" => "some-other-type",
       "reason" => "some-other-reason"
     }
@@ -208,38 +212,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass
     assert_false instance.with_transporter_log
-    assert_not_nil instance.dlq_handler
-    assert_equal 'drop', instance.dlq_handler['type']
-  end
-
-  def test_configure_with_dlq_file_handler
-    require 'tmpdir'
-    dir = Dir.mktmpdir
-    config = %Q{
-      host     logs.google.com
-      port     777
-      scheme   https
-      path     /es/
-      user     john
-      password doe
-      dlq_handler {"type":"file", "dir":"#{dir}"}
-    }
-    instance = driver('test', config).instance
-
-    assert_equal 'logs.google.com', instance.host
-    assert_equal 777, instance.port
-    assert_equal 'https', instance.scheme
-    assert_equal '/es/', instance.path
-    assert_equal 'john', instance.user
-    assert_equal 'doe', instance.password
-    assert_equal :TLSv1, instance.ssl_version
-    assert_nil instance.client_key
-    assert_nil instance.client_cert
-    assert_nil instance.client_key_pass
-    assert_false instance.with_transporter_log
-    assert_not_nil instance.dlq_handler
-    assert_equal 'file', instance.dlq_handler['type']
-    assert_true Dir.exists?(dir)
   end
 
   def test_template_already_present
@@ -629,28 +601,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
-  def test_write_message_with_dlq_drop_handler
+  def test_write_message_with_bad_chunk
     driver.configure("target_index_key bad_value\n")
     log = driver.instance.router.emit_error_handler.log
     stub_elastic_ping
     stub_elastic
     driver.emit({'bad_value'=>"\255"})
     driver.run
-    assert_logs_include(log.out.logs, 'Dropping')
-  end
-
-  def test_write_message_with_dlq_file_handler
-    log = driver.instance.router.emit_error_handler.log
-    dir = Dir.mktmpdir
-    driver.configure("dlq_handler {\"type\":\"file\", \"dir\":\"#{dir}\"}\n
-      target_index_key bad_value\n
-    ")
-    stub_elastic_ping
-    stub_elastic
-    driver.emit({'bad_value'=>"\255"})
-    driver.run
-    logs = File.readlines(File.join(dir,'dlq'))
-    assert_logs_include(logs, 'invalid')
+    assert_logs_include(log.out.logs, /(input string invalid)|(invalid byte sequence in UTF-8)/)
   end
 
   def test_writes_to_default_index
@@ -1417,25 +1375,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
-  def test_bulk_bad_arguments
-    log = driver.instance.router.emit_error_handler.log
-    log.level = 'debug'
-    driver = driver('@log_level debug')
-
-    stub_elastic_ping
-    stub_elastic_bad_argument
-
-    driver.emit(sample_record)
-    driver.emit(sample_record)
-    driver.emit(sample_record)
-    driver.run
-
-    matches = log.out.logs.grep /Elasticsearch rejected document:/
-    assert_equal(1, matches.length, "Message 'Elasticsearch rejected document: ...' was not emitted")
-    matches = log.out.logs.grep /documents due to invalid field arguments/
-    assert_equal(1, matches.length, "Message 'Elasticsearch rejected # documents due to invalid field arguments ...' was not emitted")
-  end
-
   def test_bulk_error
     stub_elastic_ping
     stub_elastic_bulk_error
@@ -1460,42 +1399,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
-  def test_bulk_error_unrecognized_error
-    stub_elastic_ping
-    stub_elastic_unrecognized_error
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::UnrecognizedElasticsearchError) {
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.run
-    }
-  end
-
-  def test_bulk_error_out_of_memory
-    stub_elastic_ping
-    stub_elastic_out_of_memory
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::ElasticsearchOutOfMemory) {
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.run
-    }
-  end
-
-  def test_bulk_error_queue_full
-    stub_elastic_ping
-    stub_elastic_bulk_rejected
-
-    assert_raise(Fluent::ElasticsearchErrorHandler::BulkIndexQueueFull) {
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.emit(sample_record)
-      driver.run
-    }
-  end
-
   def test_bulk_index_into_a_create
     stub_elastic_ping
     stub_elastic_index_to_create

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.15.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -144,8 +144,6 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
-- lib/fluent/plugin/dead_letter_queue_drop_handler.rb
-- lib/fluent/plugin/dead_letter_queue_file_handler.rb
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
@@ -153,6 +151,7 @@ files:
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
@@ -183,6 +182,7 @@ specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb

data/lib/fluent/plugin/dead_letter_queue_drop_handler.rb

@@ -1,10 +0,0 @@
-
-module Fluent::DeadLetterQueueDropHandler
-  def handle_chunk_error(out_plugin, tag, error, time, record)
-    begin
-      log.error("Dropping record from '#{tag}': error:#{error} time:#{time} record:#{record}")
-    rescue=>e
-      log.error("Error while trying to log and drop message from chunk '#{tag}' #{e.message}")
-    end
-  end
-end

data/lib/fluent/plugin/dead_letter_queue_file_handler.rb

@@ -1,14 +0,0 @@
-# encoding: UTF-8
-
-module Fluent::DeadLetterQueueFileHandler
-
-  def handle_chunk_error(out_plugin, tag, error, time, record)
-      begin
-        @dlq_file.info({processed_at: Time.now.utc, tag: tag, error: "#{error.message}", time: time, record: record}.to_json)
-      rescue=>e
-        log.error("Error while trying to log and drop message from chunk '#{tag}' #{e.message}")
-      end
-  end
-
-end
-