fluent-plugin-elasticsearch 2.11.8 → 2.11.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a1df6327a4af9049ed3fb5bbb10a153a79a183d1309996f00abcc20d8410108
-  data.tar.gz: 8129c3a35a66eca22611fdaaebe36bb9f4122ff06f3b51e1ae29675125894c88
+  metadata.gz: 85a3883a0d6e6ca5ba98a945cd6a786e2aa9e6f65e4869809e4c8ccae5f25c1d
+  data.tar.gz: 29dec4932b60be332820bb039329c173102d4f7c4e292c3ef7d4092c992d422c
 SHA512:
-  metadata.gz: 1e1c2f01409ca7f181e75c88c6b1f32d289259504a5cda50e3d99429e952b8364c2957e10aa77e2dc8f8b3c2cf7d832efe5bc88493e6537a1160b39ba1c2d833
-  data.tar.gz: baf563a878fa159061e8232a5b9916b205aa6b9d00b2a5f67d0eb1c306decab30fd7a302e8da5478eb3200c62cf4de52ccf888b7fdda75d5c3bb688a1160de93
+  metadata.gz: 243fbca142efbcca3f98933ec49fb1cb760f32dfa1c54fd1b37d5a809003bab9db296e7d5eafb8409e13617db72673579d89cd62d6abcf30d1bfefd36b4d755d
+  data.tar.gz: b37685ecda626db6426ffec6ede2139486cb34a3e83a9ef6e94b96a8609142fa9bd38f1b8b42265dce5387f992e2636583d3b1e87625f3aa4007e2157958b1ec

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 2.11.9
+- Use ConnectionRetryFailure in plugin specific retrying for consistency (#468)
+- Remove outdated generating hash_id_key code (#466)
+- Tweak behavior for UnrecoverableError and #detect_es_major_version (#465)
+
 ### 2.11.8
 - Serialize requests with Oj (#464)
 

data/README.md

@@ -715,6 +715,8 @@ The value for option `buffer_chunk_limit` should not exceed value `http.max_cont
 
 **Note**: If you use or evaluate Fluentd v0.14, you can use `<buffer>` directive to specify buffer configuration, too. In more detail, please refer to the [buffer configuration options for v0.14](https://docs.fluentd.org/v0.14/articles/buffer-plugin-overview#configuration-parameters)
 
+**Note**: If you use `disable_retry_limit` in v0.12 or `retry_forever` in v0.14 or later, please be careful to consume memory inexhaustibly.
+
 ### Hash flattening
 
 Elasticsearch will complain if you send object and concrete values to the same field. For example, you might have logs that look this, from different places:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.8'
+  s.version       = '2.11.9'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -8,12 +8,21 @@ class Fluent::Plugin::ElasticsearchErrorHandler
   attr_accessor :bulk_message_count
   class ElasticsearchVersionMismatch < Fluent::UnrecoverableError; end
   class ElasticsearchSubmitMismatch < Fluent::UnrecoverableError; end
+  class ElasticsearchRequestAbortError < Fluent::UnrecoverableError; end
   class ElasticsearchError < StandardError; end
 
   def initialize(plugin)
     @plugin = plugin
   end
 
+  def unrecoverable_error_types
+    ["out_of_memory_error", "es_rejected_execution_exception"]
+  end
+
+  def unrecoverable_error?(type)
+    unrecoverable_error_types.include?(type)
+  end
+
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
@@ -70,6 +79,9 @@ class Fluent::Plugin::ElasticsearchErrorHandler
           type = item[write_operation]['error']['type']
           stats[type] += 1
           retry_stream.add(time, rawrecord)
+          if unrecoverable_error?(type)
+            raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
+          end
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -22,7 +22,8 @@ end
 
 module Fluent::Plugin
   class ElasticsearchOutput < Output
-    class ConnectionFailure < Fluent::UnrecoverableError; end
+    class ConnectionFailure < StandardError; end
+    class ConnectionRetryFailure < Fluent::UnrecoverableError; end
 
     # MissingIdFieldError is raised for records that do not
     # include the field for the unique record identifier
@@ -199,9 +200,6 @@ EOC
         @password = URI.encode_www_form_component(m["password"])
       end
 
-      if @hash_config
-        raise Fluent::ConfigError, "@hash_config.hash_id_key and id_key must be equal." unless @hash_config.hash_id_key == @id_key
-      end
       @transport_logger = nil
       if @with_transporter_log
         @transport_logger = log
@@ -209,7 +207,13 @@ EOC
         log.warn "Consider to specify log_level with @log_level." unless log_level
       end
 
-      @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
+      @last_seen_major_version =
+        begin
+          detect_es_major_version
+        rescue ConnectionFailure
+          log.warn "Could not connect Elasticsearch. Assuming Elasticsearch 5."
+          DEFAULT_ELASTICSEARCH_VERSION
+        end
       if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
         log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
       end
@@ -528,10 +532,6 @@ EOC
         record = flatten_record(record)
       end
 
-      if @hash_config
-        record = generate_hash_id_key(record)
-      end
-
       dt = nil
       if @logstash_format || @include_timestamp
         if record.has_key?(TIMESTAMP_FIELD)
@@ -637,7 +637,7 @@ EOC
           sleep 2**retries
           retry
         end
-        raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+        raise ConnectionRetryFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
       rescue Exception
         @_es = nil if @reconnect_on_error
         @_es_info = nil if @reconnect_on_error

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -136,10 +136,6 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
-        if @hash_config
-          record = generate_hash_id_key(record)
-        end
-
         begin
           # evaluate all configurations here
           DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
@@ -238,7 +234,7 @@ module Fluent::Plugin
           sleep 2**retries
           retry
         end
-        raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+        raise ConnectionRetryFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
       rescue Exception
         @_es = nil if @reconnect_on_error
         raise

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -81,6 +81,60 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
 
+  def test_out_of_memory_responses
+    records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 500,
+            "_type"  : "bar",
+            "error" : {
+              "type" : "out_of_memory_error",
+              "reason":"Java heap space"
+            }
+          }
+        }
+      ]
+     }))
+
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    end
+  end
+
+  def test_es_rejected_execution_exception_responses
+    records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+    response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 429,
+            "_type"  : "bar",
+            "error" : {
+              "type" : "es_rejected_execution_exception",
+              "reason":"rejected execution of org.elasticsearch.transport.TransportService"
+            }
+          }
+        }
+      ]
+     }))
+
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+    assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    end
+  end
+
   def test_retry_error
     records = []
     error_records = Hash.new(false)
@@ -138,10 +192,10 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "6",
-            "status" : 429,
+            "status" : 400,
             "error" : {
-              "type" : "es_rejected_execution_exception",
-              "reason":"unable to fulfill request at this time, try again later"
+              "type" : "mapper_parsing_exception",
+              "reason":"failed to parse"
             }
           }
         },
@@ -176,19 +230,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       failed = false
       dummy_extracted_values = []
       @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
-    rescue Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
+    rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
+        next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 3, records.length
+      assert_equal 2, records.length
       assert_equal 2, records[0]['_id']
-      assert_equal 6, records[1]['_id']
-      assert_equal 8, records[2]['_id']
+      assert_equal 8, records[1]['_id']
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 2, error_ids.length
-      assert_equal 5, error_ids[0]
-      assert_equal 7, error_ids[1]
+      assert_equal 3, error_ids.length
+      assert_equal [5, 6, 7], error_ids
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end
@@ -197,4 +250,113 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   end
 
+  def test_unrecoverable_error_included_in_responses
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
+    end
+    chunk = MockChunk.new(records)
+
+    response = parse_response(%({
+      "took" : 1,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "1",
+            "status" : 201
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "2",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "3",
+            "status" : 409
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "5",
+            "status" : 500,
+            "error" : {
+              "reason":"unrecognized error - no type field"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "6",
+            "status" : 500,
+            "_type"  : "bar",
+            "error" : {
+              "type" : "out_of_memory_error",
+              "reason":"Java heap space"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "7",
+            "status" : 400,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "8",
+            "status" : 500,
+            "error" : {
+              "type" : "some unrecognized type",
+              "reason":"unrecognized error"
+            }
+          }
+        }
+      ]
+    }))
+
+    begin
+      failed = false
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+    rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        next unless e.respond_to?(:retry_stream)
+        e.retry_stream.each {|time, record| records << record}
+      end
+      # should drop entire chunk when unrecoverable error response is replied
+      assert_equal 0, records.length
+    end
+    assert_true failed
+
+  end
+
 end

data/test/plugin/test_out_elasticsearch.rb

@@ -69,6 +69,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
 
+  def stub_elastic_timeout(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).to_timeout
+  end
+
   def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
     if @index_command_counts == nil
        @index_command_counts = {}
@@ -1821,6 +1825,24 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  def test_request_forever
+    stub_elastic_ping
+    stub_elastic
+    driver.configure(Fluent::Config::Element.new(
+               'ROOT', '', {
+                 '@type' => 'elasticsearch',
+               }, [
+                 Fluent::Config::Element.new('buffer', '', {
+                                               'retry_forever' => true
+                                             }, [])
+               ]
+             ))
+    stub_elastic_timeout
+    driver.run(default_tag: 'test', timeout: 10) do
+      driver.feed(sample_record)
+    end
+  end
+
   def test_connection_failed_retry
     connection_resets = 0
 

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -62,6 +62,10 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
   end
 
+  def stub_elastic_timeout(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).to_timeout
+  end
+
   def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
     if @index_command_counts == nil
        @index_command_counts = {}
@@ -843,6 +847,24 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     }
   end
 
+  def test_request_forever
+    stub_elastic_ping
+    stub_elastic
+    driver.configure(Fluent::Config::Element.new(
+               'ROOT', '', {
+                 '@type' => 'elasticsearch',
+               }, [
+                 Fluent::Config::Element.new('buffer', '', {
+                                               'retry_forever' => true
+                                             }, [])
+               ]
+             ))
+    stub_elastic_timeout
+    driver.run(default_tag: 'test', timeout: 10) do
+      driver.feed(sample_record)
+    end
+  end
+
   def test_tag_parts_index_error_event
     stub_elastic_ping
     stub_elastic

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.8
+  version: 2.11.9
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-06 00:00:00.000000000 Z
+date: 2018-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd