logstash-output-elasticsearch 7.2.2-java → 7.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21f1b50d8253533385e29f4317d2c4354ebf6231
-  data.tar.gz: ccb251840a4bbf11a3b21bc98a83cf3aaac18198
+  metadata.gz: f92df38e9dc63ba5960b27d4f673f4a4e3914ef0
+  data.tar.gz: 3e0b56b678289dcb335e60f77f46dc860fd5a68d
 SHA512:
-  metadata.gz: f89f93ce95860b1119c274227ad8a760f67610e2adffc6f12e4cd2b5809be1e00b717ad59c1c60d849e18aea13a69426ee83828aa890dc1e233136aa842c70f9
-  data.tar.gz: a4abd5d2ad750e811d850062337d8d595f5f6e84c53dbc86b5d98878835c3db9486d9b854de52f411428f954ce901fc90d31c29333b8c0f2dd44e4be2a776caa
+  metadata.gz: 9b1fd948848fdfd1c7e6e3448ff4bf47a44b05a15042f16cf1d43e29762797aff793123e713798f6b444b135fce85e3af6deae526dc75d6e7998924efa1ba58f
+  data.tar.gz: 395df9184cb1b7b1dd7ce271dc37609ccad2ea1b7c8690e973352d3952d572e78b5053dc62b27470f56d878ea5ad2897517d6b751a26626f6f565ef27c641617

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.3.0
+ - Log 429 errors as debug instead of error. These aren't actual errors and cause users undue concern.
+   This status code is triggered when ES wants LS to backoff, which it does correctly (exponentially)
+
 ## 7.2.2
  - Docs: Add requirement to use version 6.2.5 or higher to support sending Content-Type headers.
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -4,7 +4,7 @@ module LogStash; module Outputs; class ElasticSearch;
   module Common
     attr_reader :client, :hosts
 
-    RETRYABLE_CODES = [429, 503]
+    DLQ_CODES = [400, 404]
     SUCCESS_CODES = [200, 201]
     CONFLICT_CODE = 409
 
@@ -16,6 +16,8 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def register
       @stopping = Concurrent::AtomicBoolean.new(false)
+      # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+      @dlq_writer = respond_to?(:execution_context) ? execution_context.dlq_writer : nil
       setup_hosts # properly sets @hosts
       build_client
       install_template
@@ -128,16 +130,29 @@ module LogStash; module Outputs; class ElasticSearch;
         action = actions[idx]
         action_params = action[1]
 
+        # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
+        # - For 409, we log and drop. there is nothing we can do
+        # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
+        # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
         if SUCCESS_CODES.include?(status)
           next
-        elsif CONFLICT_CODE == status && VERSION_TYPES_PERMITTING_CONFLICT.include?(action_params[:version_type])
-          @logger.debug "Ignoring external version conflict: status[#{status}] failure[#{failure}] version[#{action_params[:version]}] version_type[#{action_params[:version_type]}]"
+        elsif CONFLICT_CODE == status
+          @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
           next
-        elsif RETRYABLE_CODES.include?(status)
-          @logger.info "retrying failed action with response code: #{status} (#{failure})"
+        elsif DLQ_CODES.include?(status)
+          action_event = action[2]
+          # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+          if @dlq_writer
+            # TODO: Change this to send a map with { :status => status, :action => action } in the future
+            @dlq_writer.write(event, "Could not index event to Elasticsearch. status: #{status}, action: #{action}, response: #{response}")
+          else
+            @logger.warn "Could not index event to Elasticsearch.", status: status, action: action, response: response
+          end
+          next
+        else
+          # only log what the user whitelisted
+          @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"])
           actions_to_retry << action
-        elsif !failure_type_logging_whitelist.include?(failure["type"])
-          @logger.warn "Failed action.", status: status, action: action, response: response
         end
       end
 
@@ -232,7 +247,17 @@ module LogStash; module Outputs; class ElasticSearch;
         if RETRYABLE_CODES.include?(e.response_code)
           log_hash = {:code => e.response_code, :url => e.url.sanitized}
           log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
-          @logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
+          message = "Encountered a retryable error. Will Retry with exponential backoff "
+
+          # We treat 429s as a special case because these really aren't errors, but
+          # rather just ES telling us to back off a bit, which we do.
+          # The other retryable codes are 502 and 503, which are true errors
+          # Even though we retry the user should be made aware of these
+          if e.response_code == 429
+            @logger.debug(message, log_hash)
+          else
+            @logger.error(message, log_hash)
+          end
 
           sleep_interval = sleep_for_interval(sleep_interval)
           retry unless @stopping.true?

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -91,24 +91,9 @@ module LogStash; module Outputs; class ElasticSearch
       # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
       mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
 
-      # This plugin uses the bulk index API for improved indexing performance.
-      # This setting defines the maximum sized bulk request Logstash will make.
-      # You may want to increase this to be in line with your pipeline's batch size.
-      # If you specify a number larger than the batch size of your pipeline it will have no effect,
-      # save for the case where a filter increases the size of an inflight batch by outputting
-      # events.
-      mod.config :flush_size, :validate => :number, :deprecate => "This setting is no longer necessary as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket."
-
-      # The amount of time since last flush before a flush is forced.
-      #
-      # This setting helps ensure slow event rates don't get stuck in Logstash.
-      # For example, if your `flush_size` is 100, and you have received 10 events,
-      # and it has been more than `idle_flush_time` seconds since the last flush,
-      # Logstash will flush those 10 events automatically.
-      #
-      # This helps keep both fast and slow log streams moving along in
-      # near-real-time.
-      mod.config :idle_flush_time, :validate => :number, :default => 1
+      mod.config :flush_size, :validate => :number, :deprecated => "This setting is no longer necessary as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket."
+
+      mod.config :idle_flush_time, :validate => :number, :default => 1, :deprecated => "This is a no-op now as every pipeline batch is flushed synchronously obviating the need for this option."
 
       # Set upsert content for update mode.s
       # Create a new document with this parameter as json string if `document_id` doesn't exists

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.2.2'
+  s.version         = '7.3.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/retry_spec.rb

@@ -88,14 +88,18 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.multi_receive([event1, event1, event1, event2])
   end
 
-  it "should retry actions with response status of 429" do
-    subject.register
+  retryable_codes = [429, 502, 503]
 
-    mock_actions_with_response({"errors" => true, "statuses" => [429]},
-                               {"errors" => false})
-    expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+  retryable_codes.each do |code|
+    it "should retry actions with response status of #{code}" do
+      subject.register
 
-    subject.multi_receive([event1])
+      mock_actions_with_response({"errors" => true, "statuses" => [code]},
+                                 {"errors" => false})
+      expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+
+      subject.multi_receive([event1])
+    end
   end
 
   it "should retry an event infinitely until a non retryable status occurs" do
@@ -107,7 +111,7 @@ describe "failures in bulk class expected behavior", :integration => true do
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
-                               {"errors" => true, "statuses" => [500]})
+                               {"errors" => true, "statuses" => [400]})
 
     subject.multi_receive([event1])
   end
@@ -126,7 +130,7 @@ describe "failures in bulk class expected behavior", :integration => true do
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
                                {"errors" => true, "statuses" => [429]},
-                               {"errors" => true, "statuses" => [500]})
+                               {"errors" => true, "statuses" => [400]})
 
     subject.multi_receive([event1])
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.2.2
+  version: 7.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-28 00:00:00.000000000 Z
+date: 2017-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement