fluent-plugin-elasticsearch 2.8.6 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 282a2bfc80d90378c04ec44f035aeb941f2c98ed22b3a2f1b1add371ecf20aab
-  data.tar.gz: 6853f9d7ea05f2bc553636d597eb00d8074dbbac78d62ffca2042e35fedaefd5
+  metadata.gz: 4e6ce96c7388d21261953d86138c5f385bfb6ba79dbec118fc611d14c90a8866
+  data.tar.gz: d9df64b807a27f60ed81a1b451a7916403c6d434a5af5699006dd404ad218151
 SHA512:
-  metadata.gz: 195428011038d2a6249704c76b16bc0eb361b9f648b6c7e40cdde16ad7dee664e926e9ac71e18f840462ec5cd146a8abb039ec9ca77ad35bcf18fdff266d37dc
-  data.tar.gz: 2fb58796f3f38972b43347e9d686251722943882de537b7a206d2bba9cabc1052fc53009486e4d6a40ae19c30217dae5d2d41aaf2cdcc2226a072c6c61841850
+  metadata.gz: 8449503bc7d12055ce8c1755a2656cf86f7066666561a9ddc4c2fe3fc15ba4934f94508dc55bad42a4286af7fc7d8fb1f628554571f860640ae8b3cdf6fe0cdc
+  data.tar.gz: c803b54c1c517942cd199710ef9dde9ac428989c7e2ab29c24e02d18fd9b4187d37bf468df9a86cda295359bd9201fea79a87a197b9bcda34814bb576d9efa56

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.9.0
+- DLQ revisited v1 uplifted #398, #402 (#404)
+
 ### 2.8.6
 - auth: Fix missing auth tokens after reloading connections (#394)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.8.6'
+  s.version       = '2.9.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -16,12 +16,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
   end
 
   def handle_error(response)
-    errors = Hash.new(0)
-    errors_bad_resp = 0
-    errors_unrecognized = 0
-    successes = 0
-    duplicates = 0
-    bad_arguments = 0
+    stats = Hash.new(0)
     response['items'].each do |item|
       if item.has_key?(@plugin.write_operation)
         write_operation = @plugin.write_operation
@@ -30,7 +25,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       else
         # When we don't have an expected ops field, something changed in the API
         # expected return values (ES 2.x)
-        errors_bad_resp += 1
+        stats[:errors_bad_resp] += 1
         next
       end
       if item[write_operation].has_key?('status')
@@ -38,58 +33,44 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       else
         # When we don't have a status field, something changed in the API
         # expected return values (ES 2.x)
-        errors_bad_resp += 1
+        stats[:errors_bad_resp] += 1
         next
       end
       case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
       when CREATE_OP == write_operation && 409 == status
-        duplicates += 1
-      when 400 == status
-        bad_arguments += 1
-        @plugin.log.debug "Elasticsearch rejected document: #{item}"
-      when [429, 500].include?(status)
+        stats[:duplicates] += 1
+      else
         if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)
-          errors_bad_resp += 1
+          stats[:errors_bad_resp] += 1
           next
         end
-        errors[type] += 1
-      when [200, 201].include?(status)
-        successes += 1
-      else
-        errors_unrecognized += 1
+        stats[type] += 1
       end
     end
-    if errors_bad_resp > 0
-      msg = "Unable to parse error response from Elasticsearch, likely an API version mismatch  #{response}"
-      @plugin.log.error msg
-      raise ElasticsearchVersionMismatch, msg
-    end
-    if bad_arguments > 0
-      @plugin.log.warn "Elasticsearch rejected #{bad_arguments} documents due to invalid field arguments"
-    end
-    if duplicates > 0
-      @plugin.log.info "Encountered #{duplicates} duplicate(s) of #{successes} indexing chunk, ignoring"
-    end
-    msg = "Indexed (op = #{@plugin.write_operation}) #{successes} successfully, #{duplicates} duplicate(s), #{bad_arguments} bad argument(s), #{errors_unrecognized} unrecognized error(s)"
-    errors.each_key do |key|
-      msg << ", #{errors[key]} #{key} error(s)"
+    if stats[:errors_bad_resp] > 0
+      @plugin.log.on_debug { @plugin.log.debug("Unable to parse response from elasticsearch, likely an API version mismatch:  #{response}") }
+      raise ElasticsearchVersionMismatch, "Unable to parse error response from Elasticsearch, likely an API version mismatch. Add '@log_level debug' to your config to see the full response"
     end
-    @plugin.log.debug msg
-    if errors_unrecognized > 0
-      raise UnrecognizedElasticsearchError, "Unrecognized elasticsearch errors returned, retrying  #{response}"
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
     end
-    errors.each_key do |key|
+    stats.each_key do |key|
       case key
       when 'out_of_memory_error'
-        raise ElasticsearchOutOfMemory, "Elasticsearch has exhausted its heap, retrying"
+        raise ElasticsearchOutOfMemory, 'Elasticsearch has exhausted its heap, retrying'
       when 'es_rejected_execution_exception'
-        raise BulkIndexQueueFull, "Bulk index queue is full, retrying"
+        raise BulkIndexQueueFull, 'Bulk index queue is full, retrying'
       else
-        raise ElasticsearchError, "Elasticsearch errors returned, retrying  #{response}"
+        @plugin.log.on_debug { @plugin.log.debug("Elasticsearch errors returned, retrying:  #{response}") }
+        raise ElasticsearchError, "Elasticsearch returned errors, retrying. Add '@log_level debug' to your config to see the full response"
       end
     end
   end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -380,7 +380,7 @@ EOC
       meta = {}
 
       tag = chunk.metadata.tag
-      logstash_prefix, index_name, type_name = expand_placeholders(chunk.metadata)
+      extracted_values = expand_placeholders(chunk.metadata)
       @error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
       @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
 
@@ -388,89 +388,98 @@ EOC
         @error.records += 1
         next unless record.is_a? Hash
 
-        if @flatten_hashes
-          record = flatten_record(record)
+        begin
+          process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
         end
+      end
+      send_bulk(bulk_message) unless bulk_message.empty?
+      bulk_message.clear
+    end
 
-        if @hash_config
-          record = generate_hash_id_key(record)
-        end
+    def process_message(tag, meta, header, time, record, bulk_message, extracted_values)
+      logstash_prefix, index_name, type_name = extracted_values
 
-        dt = nil
-        if @logstash_format || @include_timestamp
-          if record.has_key?(TIMESTAMP_FIELD)
-            rts = record[TIMESTAMP_FIELD]
-            dt = parse_time(rts, time, tag)
-          elsif record.has_key?(@time_key)
-            rts = record[@time_key]
-            dt = parse_time(rts, time, tag)
-            record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision) unless @time_key_exclude_timestamp
-          else
-            dt = Time.at(time).to_datetime
-            record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
-          end
-        end
+      if @flatten_hashes
+        record = flatten_record(record)
+      end
 
-        target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
-        if target_index_parent && target_index_parent[target_index_child_key]
-          target_index = target_index_parent.delete(target_index_child_key)
-        elsif @logstash_format
-          dt = dt.new_offset(0) if @utc_index
-          target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
+      if @hash_config
+        record = generate_hash_id_key(record)
+      end
+
+      dt = nil
+      if @logstash_format || @include_timestamp
+        if record.has_key?(TIMESTAMP_FIELD)
+          rts = record[TIMESTAMP_FIELD]
+          dt = parse_time(rts, time, tag)
+        elsif record.has_key?(@time_key)
+          rts = record[@time_key]
+          dt = parse_time(rts, time, tag)
+          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision) unless @time_key_exclude_timestamp
         else
-          target_index = index_name
+          dt = Time.at(time).to_datetime
+          record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
         end
+      end
 
-        # Change target_index to lower-case since Elasticsearch doesn't
-        # allow upper-case characters in index names.
-        target_index = target_index.downcase
-        if @include_tag_key
-          record[@tag_key] = tag
-        end
+      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+      if target_index_parent && target_index_parent[target_index_child_key]
+        target_index = target_index_parent.delete(target_index_child_key)
+      elsif @logstash_format
+        dt = dt.new_offset(0) if @utc_index
+        target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
+      else
+        target_index = index_name
+      end
 
-        target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
-        if target_type_parent && target_type_parent[target_type_child_key]
-          target_type = target_type_parent.delete(target_type_child_key)
-          if @last_seen_major_version == 6
-            log.warn "Detected ES 6.x: `@type_name` will be used as the document `_type`."
-            target_type = type_name
-          elsif @last_seen_major_version >= 7
-            log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
-            target_type = '_doc'.freeze
-          end
+      # Change target_index to lower-case since Elasticsearch doesn't
+      # allow upper-case characters in index names.
+      target_index = target_index.downcase
+      if @include_tag_key
+        record[@tag_key] = tag
+      end
+
+      target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
+      if target_type_parent && target_type_parent[target_type_child_key]
+        target_type = target_type_parent.delete(target_type_child_key)
+        if @last_seen_major_version == 6
+          log.warn "Detected ES 6.x: `@type_name` will be used as the document `_type`."
+          target_type = type_name
+        elsif @last_seen_major_version >= 7
+          log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+          target_type = '_doc'.freeze
+        end
+      else
+        if @last_seen_major_version >= 7 && target_type != DEFAULT_TYPE_NAME_ES_7x
+          log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
+          target_type = '_doc'.freeze
         else
-          if @last_seen_major_version >= 7 && target_type != DEFAULT_TYPE_NAME_ES_7x
-            log.warn "Detected ES 7.x or above: `_doc` will be used as the document `_type`."
-            target_type = '_doc'.freeze
-          else
-            target_type = type_name
-          end
+          target_type = type_name
         end
+      end
 
-        meta.clear
-        meta["_index".freeze] = target_index
-        meta["_type".freeze] = target_type
-
-        if @pipeline
-          meta["pipeline".freeze] = @pipeline
-        end
+      meta.clear
+      meta["_index".freeze] = target_index
+      meta["_type".freeze] = target_type
 
-        @meta_config_map.each do |record_accessor, meta_key|
-          if raw_value = record_accessor.call(record)
-            meta[meta_key] = raw_value
-          end
-        end
+      if @pipeline
+        meta["pipeline".freeze] = @pipeline
+      end
 
-        if @remove_keys
-          @remove_keys.each { |key| record.delete(key) }
+      @meta_config_map.each do |record_accessor, meta_key|
+        if raw_value = record_accessor.call(record)
+          meta[meta_key] = raw_value
         end
+      end
 
-        append_record_to_messages(@write_operation, meta, header, record, bulk_message)
-        @error.bulk_message_count += 1
+      if @remove_keys
+        @remove_keys.each { |key| record.delete(key) }
       end
 
-      send_bulk(bulk_message) unless bulk_message.empty?
-      bulk_message.clear
+      append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+      @error.bulk_message_count += 1
     end
 
     # returns [parent, child_key] of child described by path array in record's tree
@@ -484,10 +493,7 @@ EOC
       retries = 0
       begin
         response = client.bulk body: data
-        if response['errors']
-          @error.handle_error(response)
-          log.error "Could not push log to Elasticsearch: #{response}"
-        end
+        @error.handle_error(response) if response['errors']
       rescue *client.transport.host_unreachable_exceptions => e
         if retries < 2
           retries += 1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -0,0 +1,122 @@
+require 'helper'
+require 'fluent/plugin/elasticsearch_error_handler'
+require 'json'
+
+class TestElasticsearchErrorHandler < Test::Unit::TestCase
+
+  class TestPlugin
+    attr_reader :log
+    def initialize(log)
+      @log = log
+    end
+
+    def write_operation
+      'index'
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+    @log = Fluent::Engine.log
+    plugin = TestPlugin.new(@log)
+    @handler =  Fluent::Plugin::ElasticsearchErrorHandler.new(plugin)
+  end
+
+  def parse_response(value)
+    JSON.parse(value)
+  end
+
+  def test_errors
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 400,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        }
+      ]
+    }))
+
+    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError do
+        @handler.handle_error(response)
+    end
+
+  end
+
+  def test_elasticsearch_version_mismatch_raises_error
+    response = parse_response(%(
+      {
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "abc",
+            "status" : 500,
+            "error" : {
+              "reason":"some error to cause version mismatch"
+            }
+          }
+        }
+      ]
+      }
+    ))
+
+    assert_raise Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchVersionMismatch do
+        @handler.handle_error(response)
+    end
+
+  end
+
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -154,6 +154,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "es_rejected_execution_exception",
       "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
     }
@@ -162,6 +163,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "out_of_memory_error",
       "reason" => "Java heap space"
     }
@@ -170,6 +172,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_unrecognized_error(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "type" => "some-other-type",
       "reason" => "some-other-reason"
     }
@@ -178,6 +181,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic_version_mismatch(url="http://localhost:9200/_bulk")
     error = {
+      "status" => 500,
       "category" => "some-other-type",
       "reason" => "some-other-reason"
     }
@@ -201,6 +205,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
   end
 
+  def assert_logs_include(logs, msg)
+    matches = logs.grep /#{msg}/
+    assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
+  end
+
   def test_configure
     config = %{
       host     logs.google.com
@@ -660,6 +669,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(elastic_request)
   end
 
+  def test_write_message_with_bad_chunk
+    driver.configure("target_index_key bad_value\n@log_level debug\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed({'bad_value'=>"\255"})
+    end
+    error_log = driver.error_events.map {|e| e.last.message }
+
+    assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
+  end
+
   def test_writes_to_default_index
     stub_elastic_ping
     stub_elastic
@@ -1724,23 +1745,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 1)
   end
 
-  def test_bulk_bad_arguments
-    driver = driver('@log_level debug')
-
-    stub_elastic_ping
-    stub_elastic_bad_argument
-
-    driver.run(default_tag: 'test', shutdown: false) do
-      driver.feed(sample_record)
-      driver.feed(sample_record)
-      driver.feed(sample_record)
-    end
-
-    matches = driver.logs.grep /Elasticsearch rejected document:/
-    assert_equal(1, matches.length, "Message 'Elasticsearch rejected document: ...' was not emitted")
-    matches = driver.logs.grep /documents due to invalid field arguments/
-    assert_equal(1, matches.length, "Message 'Elasticsearch rejected # documents due to invalid field arguments ...' was not emitted")
-  end
 
   def test_bulk_error
     stub_elastic_ping
@@ -1768,45 +1772,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
-  def test_bulk_error_unrecognized_error
-    stub_elastic_ping
-    stub_elastic_unrecognized_error
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::UnrecognizedElasticsearchError) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-      end
-    }
-  end
-
-  def test_bulk_error_out_of_memory
-    stub_elastic_ping
-    stub_elastic_out_of_memory
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchOutOfMemory) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-      end
-    }
-  end
-
-  def test_bulk_error_queue_full
-    stub_elastic_ping
-    stub_elastic_bulk_rejected
-
-    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::BulkIndexQueueFull) {
-      driver.run(default_tag: 'test', shutdown: false) do
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-        driver.feed(sample_record)
-      end
-    }
-  end
-
   def test_bulk_index_into_a_create
     stub_elastic_ping
     stub_elastic_index_to_create

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.8.6
+  version: 2.9.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -151,6 +151,7 @@ files:
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
@@ -181,6 +182,7 @@ specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb