logstash-output-elasticsearch 7.3.8-java → 7.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 00d19efacd7374a6f3d095ac5a8b5d76d3357ac0
-  data.tar.gz: 08e21adfff2a195a0d0c6cbab1b4051ee4df39e5
+SHA256:
+  metadata.gz: 868ec97797a9b0bf6a6d57b805ce4097c937ab6c30bb0b2f805cd809e0c6c685
+  data.tar.gz: b91cdaf715b3cfae6836aeeb780e7a7a52fd6ee9fdfd4b182fc1596e238b88e8
 SHA512:
-  metadata.gz: df592abc3005cf4e81c5869ecf2705afd2ee4aefd4fb95330c1ff876677ca4862f7032c99878a8f7ecd2dedf3cfbdbf7b767e26083ac271895e502f63af0d464
-  data.tar.gz: 4f42e2dc5518590c4765a9d1c4f7d1a63870ca8b371bdaca7e2a7e98e9d7e9372a087be189316115a0b898723fad38bbfdb1a1caacfd66226437416bbdf697b9
+  metadata.gz: bdc11c661ef02325e7b984145244acd4f8df9842e4009928a8a4a9d6d9d2bae288a6aed8047fc15ea0cbf3a096c6211fca1c50aa1dbf191a97d6d96f18bd0f44
+  data.tar.gz: 066b2f9fa955dc737495ef330a0f75473ede71c7fd3bc5b265f60767dbc0087bd02d247d00dc435b0542a698d90421edcd5ddf9ecd275f6e61351b5e0dbe1b78

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.4.0
+  - Retry all non-200 responses of the bulk API indefinitely
+  - Improve documentation on retry codes
+
 ##  7.3.8
  - Fix bug where java class names were logged rather than actual host names in various scenarios
 

data/docs/index.asciidoc

@@ -55,17 +55,18 @@ the new template is installed.
 
 ==== Retry Policy
 
-The retry policy has changed significantly in the 2.2.0 release.
+The retry policy has changed significantly in the 7.4.0 release.
 This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
-either partial or total failures.
+either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
+request are handled differently than error codes for individual documents.
 
-The following errors are retried infinitely:
+HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
 
-- Network errors (inability to connect)
-- 429 (Too many requests) and
-- 503 (Service unavailable) errors
+The following document errors are handled as follows:
+- 400 and 404 errors are sent to the DLQ if enabled. If a DLQ is not enabled a log message will be emitted and the event will be dropped.
+- 409 errors (conflict) are logged as a warning and dropped. 
 
-NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
+Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
 
 ==== Batch Sizes ====

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -4,14 +4,10 @@ module LogStash; module Outputs; class ElasticSearch;
   module Common
     attr_reader :client, :hosts
 
-    # These are codes for temporary recoverable conditions
-    # 429 just means that ES has too much traffic ATM
-    # 503 means it , or a proxy is temporarily unavailable
-    RETRYABLE_CODES = [429, 503]
-
-    DLQ_CODES = [400, 404]
-    SUCCESS_CODES = [200, 201]
-    CONFLICT_CODE = 409
+    # These codes apply to documents, not at the request level
+    DOC_DLQ_CODES = [400, 404]
+    DOC_SUCCESS_CODES = [200, 201]
+    DOC_CONFLICT_CODE = 409
 
     # When you use external versioning, you are communicating that you want
     # to ignore conflicts. More obviously, since an external version is a 
@@ -140,12 +136,12 @@ module LogStash; module Outputs; class ElasticSearch;
         # - For 409, we log and drop. there is nothing we can do
         # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
         # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
-        if SUCCESS_CODES.include?(status)
+        if DOC_SUCCESS_CODES.include?(status)
           next
-        elsif CONFLICT_CODE == status
+        elsif DOC_CONFLICT_CODE == status
           @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
           next
-        elsif DLQ_CODES.include?(status)
+        elsif DOC_DLQ_CODES.include?(status)
           action_event = action[2]
           # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
           if @dlq_writer
@@ -250,29 +246,22 @@ module LogStash; module Outputs; class ElasticSearch;
         sleep_interval = next_sleep_interval(sleep_interval)
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        if RETRYABLE_CODES.include?(e.response_code)
-          log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
-          log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
-          message = "Encountered a retryable error. Will Retry with exponential backoff "
-
-          # We treat 429s as a special case because these really aren't errors, but
-          # rather just ES telling us to back off a bit, which we do.
-          # The other retryable code is 503, which are true errors
-          # Even though we retry the user should be made aware of these
-          if e.response_code == 429
-            logger.debug(message, log_hash)
-          else
-            logger.error(message, log_hash)
-          end
-
-          sleep_interval = sleep_for_interval(sleep_interval)
-          retry
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
+        message = "Encountered a retryable error. Will Retry with exponential backoff "
+
+        # We treat 429s as a special case because these really aren't errors, but
+        # rather just ES telling us to back off a bit, which we do.
+        # The other retryable code is 503, which are true errors
+        # Even though we retry the user should be made aware of these
+        if e.response_code == 429
+          logger.debug(message, log_hash)
         else
-          log_hash = {:code => e.response_code, 
-                      :response_body => e.response_body}
-          log_hash[:request_body] = e.request_body if @logger.debug?
-          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", log_hash)
+          logger.error(message, log_hash)
         end
+
+        sleep_interval = sleep_for_interval(sleep_interval)
+        retry
       rescue => e
         # Stuff that should never happen
         # For all other errors print out full connection issues
@@ -285,7 +274,6 @@ module LogStash; module Outputs; class ElasticSearch;
 
         @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
 
-        # We retry until there are no errors! Errors should all go to the retry queue
         sleep_interval = sleep_for_interval(sleep_interval)
         retry unless @stopping.true?
       end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -140,6 +140,13 @@ module LogStash; module Outputs; class ElasticSearch;
         body_stream.truncate(0)
         body_stream.seek(0)
       end
+
+      if response.code != 200
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+          response.code, @bulk_path, body_stream.to_s, response.body
+        )
+      end
+
       LogStash::Json.load(response.body)
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.3.8'
+  s.version         = '7.4.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -212,8 +212,10 @@ describe "outputs/elasticsearch" do
         )
       end
       let(:logger) { double("logger").as_null_object }
+      let(:response) { { :errors => [], :items => [] } }
 
       before(:each) do
+
         i = 0
         bulk_param =  [["index", anything, event.to_hash]]
 
@@ -225,7 +227,7 @@ describe "outputs/elasticsearch" do
           if i == 1
             raise error
           end
-        end
+        end.and_return(response)
         eso.multi_receive([event])
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.3.8
+  version: 7.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-20 00:00:00.000000000 Z
+date: 2017-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -196,7 +196,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -280,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: Logstash Output to Elasticsearch