logstash-output-http 5.4.0 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd850256e5336762a98ed5cc4ae699ddcc1c3c3bcb9e5382b033aa21aebd5fea
-  data.tar.gz: '08b1450d6eadd6ea8e7a9451d781c4fd78c3d7b199751616c1e74cabc75e3899'
+  metadata.gz: 6f8c174b5f5725b3dff206924edcff21ebc71899e386c1a301860dba438bbec5
+  data.tar.gz: dd6f158c15357a1dee6bfc3ad6e0faa4f6080c3fc2a89b6050b188bd3d016d4a
 SHA512:
-  metadata.gz: 11dce3822d80720abedca11fe586cb8d4c8384e44cb793da13bba509c571c48bf8b617a0545297fb49a8cd254d5374e5b4702dddf607e4bad04eaaf07b24e0d0
-  data.tar.gz: 62546e31fc1386c48a0b77c25f5e13093fc63bfc371ea1924ebc8737ec26b4855c4c0bd40cebbd5a2eac82288315da436c46772793c2fc86adcf848d79cfadd9
+  metadata.gz: 2d668381891939636b2462361fe0011d5ced8461b5a1a5e7f2662aa765e4672b6377e2dd37a05073eaa8e0602d60632496fa48ec9c42ab5063839f02214f2e9d
+  data.tar.gz: b3ff1de393aee131f18a1d8d75f05389f6a74642e695626b1e9b4b03eec99b7a26017cea50b97315f35594013650785a8b3e11bfcc7acd4e5d32ed118340f208

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.4.1
+  - Fix retry indefinitely in termination process. This feature requires Logstash 8.1 [#129](https://github.com/logstash-plugins/logstash-output-http/pull/129)
+  - Docs: Add retry policy description [#130](https://github.com/logstash-plugins/logstash-output-http/pull/130)
+
 ## 5.4.0
   - Introduce retryable unknown exceptions for "connection reset by peer" and "timeout" [#127](https://github.com/logstash-plugins/logstash-output-http/pull/127)
 

data/docs/index.asciidoc

@@ -31,6 +31,40 @@ guaranteed!
 
 Beware, this gem does not yet support codecs. Please use the 'format' option for now.
 
+[id="plugins-{type}s-{plugin}-retry_policy"]
+==== Retry policy
+
+This output has two levels of retry: library and plugin.
+
+[id="plugins-{type}s-{plugin}-library_retry"]
+===== Library retry
+
+The library retry applies to IO related failures. 
+Non retriable errors include SSL related problems, unresolvable hosts,
+connection issues, and OS/JVM level interruptions happening during a request.
+
+The options for library retry are: 
+
+* <<plugins-{type}s-{plugin}-automatic_retries,`automatic_retries`>>. 
+Controls the number of times the plugin should retry after failures at the library level.
+* <<plugins-{type}s-{plugin}-retry_non_idempotent,`retry_non_idempotent`>>. 
+When set to `false`, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be
+retried.
+
+[id="plugins-{type}s-{plugin}-plugin_retry"]
+===== Plugin retry
+
+The options for plugin level retry are: 
+
+* <<plugins-{type}s-{plugin}-retry_failed,`retry_failed`>>. 
+When set to `true`, the plugin retries indefinitely for HTTP error response codes defined
+in the <<plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> option
+(429, 500, 502, 503, 504) and retryable exceptions (socket timeout/ error, DNS resolution failure and client protocol exception).
+* <<plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>>. 
+Sets http response codes that trigger a retry. 
+
+NOTE: The `retry_failed` option does not control the library level retry.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Http Output Configuration Options
 
@@ -85,12 +119,10 @@ output plugins.
   * Value type is <<number,number>>
   * Default value is `1`
 
-How many times should the client retry a failing URL. We highly recommend NOT setting this value
-to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
-Only IO related failures will be retried, such as connection timeouts and unreachable hosts.
-Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting,
-unless `retry_failed` is set.
-Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
+How many times should the client retry a failing URL. We recommend setting this option
+to a value other than zero if the <<plugins-{type}s-{plugin}-keepalive,`keepalive` option>> is enabled. 
+Some servers incorrectly end keepalives early, requiring a retry.
+See <<plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information.
 
 [id="plugins-{type}s-{plugin}-cacert"]
 ===== `cacert` 
@@ -311,7 +343,12 @@ Timeout (in seconds) for the entire request
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
-Set this to false if you don't want this output to retry failed requests
+Note that this option controls plugin-level retries only. 
+It has no affect on library-level retries. 
+
+Set this option to `false` if you want to disable infinite retries for HTTP error response codes defined in the <<plugins-{type}s-{plugin}-retryable_codes,`retryable_codes`>> or 
+retryable exceptions (Timeout, SocketException, ClientProtocolException, ResolutionFailure and SocketTimeout).
+See <<plugins-{type}s-{plugin}-retry_policy,Retry policy>> for more information.
 
 [id="plugins-{type}s-{plugin}-retry_non_idempotent"]
 ===== `retry_non_idempotent` 
@@ -319,8 +356,10 @@ Set this to false if you don't want this output to retry failed requests
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
-This only affects connectivity related errors (see related `automatic_retries` setting).
+When this option is set to `false` and `automatic_retries` is enabled, GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
+
+When set to `true` and `automatic_retries` is enabled, this will cause non-idempotent HTTP verbs (such as POST) to be retried.
+See <<plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information.
 
 [id="plugins-{type}s-{plugin}-retryable_codes"]
 ===== `retryable_codes` 
@@ -328,7 +367,8 @@ This only affects connectivity related errors (see related `automatic_retries` s
   * Value type is <<number,number>>
   * Default value is `[429, 500, 502, 503, 504]`
 
-If encountered as response codes this plugin will retry these requests
+If the plugin encounters these response codes, the plugin will retry indefinitely.
+See <<plugins-{type}s-{plugin}-retry_policy,Retry Policy>> for more information.
 
 [id="plugins-{type}s-{plugin}-socket_timeout"]
 ===== `socket_timeout` 
@@ -405,4 +445,4 @@ See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/outputs/http.rb

@@ -28,6 +28,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     /Read Timed out/i
   ]
 
+  class PluginInternalQueueLeftoverError < StandardError; end
 
   # This output lets you send events to a
   # generic HTTP(S) endpoint
@@ -179,6 +180,9 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
 
       event, attempt = popped
 
+      raise PluginInternalQueueLeftoverError.new("Received pipeline shutdown request but http output has unfinished events. " \
+              "If persistent queue is enabled, events will be retried.") if attempt > 2 && pipeline_shutdown_requested?
+
       action, event, attempt = send_event(event, attempt)
       begin
         action = :failure if action == :retry && !@retry_failed
@@ -223,6 +227,11 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     raise e
   end
 
+  def pipeline_shutdown_requested?
+    return super if defined?(super) # since LS 8.1.0
+    nil
+  end
+
   def sleep_for_attempt(attempt)
     sleep_for = attempt**2
     sleep_for = sleep_for <= 60 ? sleep_for : 60

data/logstash-output-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-http'
-  s.version         = '5.4.0'
+  s.version         = '5.4.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Sends events to a generic HTTP or HTTPS endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/http_spec.rb

@@ -501,6 +501,23 @@ describe LogStash::Outputs::Http do
       let(:base_config) { { "http_compression" => true } }
     end
   end
+
+  describe "retryable error in termination" do
+    let(:url) { "http://localhost:#{port-1}/invalid" }
+    let(:events) { [event] }
+    let(:config) { {"url" => url, "http_method" => "get", "pool_max" => 1} }
+
+    subject { LogStash::Outputs::Http.new(config) }
+
+    before do
+      subject.register
+      allow(subject).to receive(:pipeline_shutdown_requested?).and_return(true)
+    end
+
+    it "raise exception to exit indefinitely retry" do
+      expect { subject.multi_receive(events) }.to raise_error(LogStash::Outputs::Http::PluginInternalQueueLeftoverError)
+    end
+  end
 end
 
 describe LogStash::Outputs::Http do # different block as we're starting web server with TLS

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-http
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 5.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-09 00:00:00.000000000 Z
+date: 2022-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement