logstash-output-http 5.2.2 → 5.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bd00218e30610f309f7870230ffeff01b5d1481b38d8723cb2e938f2067a561b
4
- data.tar.gz: 6c31789a86e548649b8ae995cab4e5927fe071d4491570d98c3ef454c28649fd
3
+ metadata.gz: 58ad383035622e8cc13441e89147871f55669f22ff22835a0d6db9a87d1ab185
4
+ data.tar.gz: 28498a16c1ee3fefcf87cb6990ca99a90be82cb3e73438dc8a564d171a3bcc9f
5
5
  SHA512:
6
- metadata.gz: 0ed20344862d2b1587d8a3b5cb8bb6b71de3c51ac75a91c85f5f1326185438f899ff08573de4613394c0fbae252c572e8c4996f05497e9cf081fc92c227182c1
7
- data.tar.gz: 11248626b51ec811d865f761e01a7adf219b9967211c20abd71c901ff16d0e551201c7d931d209f19cf40fa6d7dad164e4dbe06f2c13ee2bfa7549bb8a16b1b3
6
+ metadata.gz: a14293accaea16058469ad4e2c92b759b8686fee8b068f31dd3d8984679260cb32c080287f08e02e9c2ac60ca84c8129fbe042eb76ca69333959cc07da50d6ae
7
+ data.tar.gz: e92646b17dbe8662d707526cb983f898bfa80392662bfe0658ad8abb80e0b786acbc24e73bce98e6621a8b99b75e90bede275cd7162da2a3b9137e93954525dc
data/CHANGELOG.md CHANGED
@@ -1,3 +1,15 @@
1
+ ## 5.3.0
2
+ - Feat: support ssl_verification_mode option [#126](https://github.com/logstash-plugins/logstash-output-http/pull/126)
3
+
4
+ ## 5.2.5
5
+ - Reduce amount of default logging on a failed request [#122](https://github.com/logstash-plugins/logstash-output-http/pull/122)
6
+
7
+ ## 5.2.4
8
+ - Relax dependency on http_client mixin since current major works on both
9
+
10
+ ## 5.2.3
11
+ - Fixed handling of empty `retryable_codes` [#99](https://github.com/logstash-plugins/logstash-output-http/pull/99)
12
+
1
13
  ## 5.2.2
2
14
  - Fix high CPU usage on retries in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
3
15
  - Enable compression in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
data/LICENSE CHANGED
@@ -1,13 +1,202 @@
1
- Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
2
1
 
3
- Licensed under the Apache License, Version 2.0 (the "License");
4
- you may not use this file except in compliance with the License.
5
- You may obtain a copy of the License at
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
6
5
 
7
- http://www.apache.org/licenses/LICENSE-2.0
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8
7
 
9
- Unless required by applicable law or agreed to in writing, software
10
- distributed under the License is distributed on an "AS IS" BASIS,
11
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- See the License for the specific language governing permissions and
13
- limitations under the License.
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright 2020 Elastic and contributors
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-http.svg)](https://travis-ci.org/logstash-plugins/logstash-output-http)
3
+ [![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-http.svg)](https://travis-ci.com/logstash-plugins/logstash-output-http)
4
4
 
5
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
6
6
 
data/docs/index.asciidoc CHANGED
@@ -66,6 +66,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
66
66
  | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
67
67
  | <<plugins-{type}s-{plugin}-retryable_codes>> |<<number,number>>|No
68
68
  | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
69
+ | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
69
70
  | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
70
71
  | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
71
72
  | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
@@ -86,7 +87,10 @@ output plugins.
86
87
 
87
88
  How many times should the client retry a failing URL. We highly recommend NOT setting this value
88
89
  to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
89
- Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
90
+ Only IO related failures will be retried, such as connection timeouts and unreachable hosts.
91
+ Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting,
92
+ unless `retry_failed` is set.
93
+ Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
90
94
 
91
95
  [id="plugins-{type}s-{plugin}-cacert"]
92
96
  ===== `cacert`
@@ -316,6 +320,7 @@ Set this to false if you don't want this output to retry failed requests
316
320
  * Default value is `false`
317
321
 
318
322
  If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
323
+ This only affects connectivity related errors (see related `automatic_retries` setting).
319
324
 
320
325
  [id="plugins-{type}s-{plugin}-retryable_codes"]
321
326
  ===== `retryable_codes`
@@ -333,6 +338,22 @@ If encountered as response codes this plugin will retry these requests
333
338
 
334
339
  Timeout (in seconds) to wait for data on the socket. Default is `10s`
335
340
 
341
+ [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
342
+ ===== `ssl_verification_mode`
343
+
344
+ * Value type is <<string,string>>
345
+ * Supported values are: `full`, `none`
346
+ * Default value is `full`
347
+
348
+ Controls the verification of server certificates.
349
+ The `full` option verifies that the provided certificate is signed by a trusted authority (CA)
350
+ and also that the server’s hostname (or IP address) matches the names identified within the certificate.
351
+
352
+ The `none` setting performs no verification of the server’s certificate.
353
+ This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration.
354
+ It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors.
355
+ Using `none` in production environments is strongly discouraged.
356
+
336
357
  [id="plugins-{type}s-{plugin}-truststore"]
337
358
  ===== `truststore`
338
359
 
@@ -384,4 +405,4 @@ See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache
384
405
  [id="plugins-{type}s-{plugin}-common-options"]
385
406
  include::{include_path}/{type}.asciidoc[]
386
407
 
387
- :default_codec!:
408
+ :default_codec!:
@@ -8,18 +8,18 @@ require "zlib"
8
8
 
9
9
  class LogStash::Outputs::Http < LogStash::Outputs::Base
10
10
  include LogStash::PluginMixins::HttpClient
11
-
11
+
12
12
  concurrency :shared
13
13
 
14
14
  attr_accessor :is_batch
15
15
 
16
16
  VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
17
-
17
+
18
18
  RETRYABLE_MANTICORE_EXCEPTIONS = [
19
19
  ::Manticore::Timeout,
20
20
  ::Manticore::SocketException,
21
- ::Manticore::ClientProtocolException,
22
- ::Manticore::ResolutionFailure,
21
+ ::Manticore::ClientProtocolException,
22
+ ::Manticore::ResolutionFailure,
23
23
  ::Manticore::SocketTimeout
24
24
  ]
25
25
 
@@ -53,14 +53,14 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
53
53
  # * if format is "json", "application/json"
54
54
  # * if format is "form", "application/x-www-form-urlencoded"
55
55
  config :content_type, :validate => :string
56
-
56
+
57
57
  # Set this to false if you don't want this output to retry failed requests
58
58
  config :retry_failed, :validate => :boolean, :default => true
59
-
59
+
60
60
  # If encountered as response codes this plugin will retry these requests
61
61
  config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
62
-
63
- # If you would like to consider some non-2xx codes to be successes
62
+
63
+ # If you would like to consider some non-2xx codes to be successes
64
64
  # enumerate them here. Responses returning these codes will be considered successes
65
65
  config :ignorable_codes, :validate => :number, :list => true
66
66
 
@@ -85,7 +85,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
85
85
 
86
86
  # Set this to true if you want to enable gzip compression for your http requests
87
87
  config :http_compression, :validate => :boolean, :default => false
88
-
88
+
89
89
  config :message, :validate => :string
90
90
 
91
91
  def register
@@ -114,7 +114,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
114
114
  @headers["Content-Type"] = @content_type
115
115
 
116
116
  validate_format!
117
-
117
+
118
118
  # Run named Timer as daemon thread
119
119
  @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
120
120
  end # def register
@@ -123,7 +123,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
123
123
  return if events.empty?
124
124
  send_events(events)
125
125
  end
126
-
126
+
127
127
  class RetryTimerTask < java.util.TimerTask
128
128
  def initialize(pending, event, attempt)
129
129
  @pending = pending
@@ -131,17 +131,18 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
131
131
  @attempt = attempt
132
132
  super()
133
133
  end
134
-
134
+
135
135
  def run
136
136
  @pending << [@event, @attempt]
137
137
  end
138
138
  end
139
139
 
140
140
  def log_retryable_response(response)
141
+ retry_msg = @retry_failed ? 'will retry' : "won't retry"
141
142
  if (response.code == 429)
142
- @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
143
+ @logger.debug? && @logger.debug("Encountered a 429 response, #{retry_msg}. This is not serious, just flow control via HTTP")
143
144
  else
144
- @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
145
+ @logger.warn("Encountered a retryable HTTP request in HTTP output, #{retry_msg}", :code => response.code, :body => response.body)
145
146
  end
146
147
  end
147
148
 
@@ -153,7 +154,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
153
154
  :event => event
154
155
  )
155
156
  end
156
-
157
+
157
158
  def send_events(events)
158
159
  successes = java.util.concurrent.atomic.AtomicInteger.new(0)
159
160
  failures = java.util.concurrent.atomic.AtomicInteger.new(0)
@@ -169,44 +170,43 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
169
170
 
170
171
  while popped = pending.pop
171
172
  break if popped == :done
172
-
173
+
173
174
  event, attempt = popped
174
175
 
175
- send_event(event, attempt) do |action,event,attempt|
176
- begin
177
- action = :failure if action == :retry && !@retry_failed
178
-
179
- case action
180
- when :success
181
- successes.incrementAndGet
182
- when :retry
183
- retries.incrementAndGet
184
-
185
- next_attempt = attempt+1
186
- sleep_for = sleep_for_attempt(next_attempt)
187
- @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
188
- timer_task = RetryTimerTask.new(pending, event, next_attempt)
189
- @timer.schedule(timer_task, sleep_for*1000)
190
- when :failure
191
- failures.incrementAndGet
192
- else
193
- raise "Unknown action #{action}"
194
- end
195
-
196
- if action == :success || action == :failure
197
- if successes.get+failures.get == event_count
198
- pending << :done
199
- end
200
- end
201
- rescue => e
202
- # This should never happen unless there's a flat out bug in the code
203
- @logger.error("Error sending HTTP Request",
204
- :class => e.class.name,
205
- :message => e.message,
206
- :backtrace => e.backtrace)
176
+ action, event, attempt = send_event(event, attempt)
177
+ begin
178
+ action = :failure if action == :retry && !@retry_failed
179
+
180
+ case action
181
+ when :success
182
+ successes.incrementAndGet
183
+ when :retry
184
+ retries.incrementAndGet
185
+
186
+ next_attempt = attempt+1
187
+ sleep_for = sleep_for_attempt(next_attempt)
188
+ @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
189
+ timer_task = RetryTimerTask.new(pending, event, next_attempt)
190
+ @timer.schedule(timer_task, sleep_for*1000)
191
+ when :failure
207
192
  failures.incrementAndGet
208
- raise e
193
+ else
194
+ raise "Unknown action #{action}"
209
195
  end
196
+
197
+ if action == :success || action == :failure
198
+ if successes.get+failures.get == event_count
199
+ pending << :done
200
+ end
201
+ end
202
+ rescue => e
203
+ # This should never happen unless there's a flat out bug in the code
204
+ @logger.error("Error sending HTTP Request",
205
+ :class => e.class.name,
206
+ :message => e.message,
207
+ :backtrace => e.backtrace)
208
+ failures.incrementAndGet
209
+ raise e
210
210
  end
211
211
  end
212
212
  rescue => e
@@ -216,13 +216,13 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
216
216
  :backtrace => e.backtrace)
217
217
  raise e
218
218
  end
219
-
219
+
220
220
  def sleep_for_attempt(attempt)
221
221
  sleep_for = attempt**2
222
222
  sleep_for = sleep_for <= 60 ? sleep_for : 60
223
223
  (sleep_for/2) + (rand(0..sleep_for)/2)
224
224
  end
225
-
225
+
226
226
  def send_event(event, attempt)
227
227
  body = event_body(event)
228
228
 
@@ -237,62 +237,44 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
237
237
  end
238
238
 
239
239
  # Create an async request
240
- request = client.background.send(@http_method, url, :body => body, :headers => headers)
241
-
242
- request.on_success do |response|
243
- begin
244
- if !response_success?(response)
245
- if retryable_response?(response)
246
- log_retryable_response(response)
247
- yield :retry, event, attempt
248
- else
249
- log_error_response(response, url, event)
250
- yield :failure, event, attempt
251
- end
252
- else
253
- yield :success, event, attempt
254
- end
255
- rescue => e
256
- # Shouldn't ever happen
257
- @logger.error("Unexpected error in request success!",
258
- :class => e.class.name,
259
- :message => e.message,
260
- :backtrace => e.backtrace)
240
+ response = client.send(@http_method, url, :body => body, :headers => headers).call
241
+
242
+ if !response_success?(response)
243
+ if retryable_response?(response)
244
+ log_retryable_response(response)
245
+ return :retry, event, attempt
246
+ else
247
+ log_error_response(response, url, event)
248
+ return :failure, event, attempt
261
249
  end
250
+ else
251
+ return :success, event, attempt
262
252
  end
263
253
 
264
- request.on_failure do |exception|
265
- begin
266
- will_retry = retryable_exception?(exception)
267
- log_failure("Could not fetch URL",
268
- :url => url,
269
- :method => @http_method,
270
- :body => body,
271
- :headers => headers,
272
- :message => exception.message,
273
- :class => exception.class.name,
274
- :backtrace => exception.backtrace,
275
- :will_retry => will_retry
276
- )
277
-
278
- if will_retry
279
- yield :retry, event, attempt
280
- else
281
- yield :failure, event, attempt
282
- end
283
- rescue => e
284
- # Shouldn't ever happen
285
- @logger.error("Unexpected error in request failure!",
286
- :class => e.class.name,
287
- :message => e.message,
288
- :backtrace => e.backtrace)
289
- end
254
+ rescue => exception
255
+ will_retry = retryable_exception?(exception)
256
+ log_params = {
257
+ :url => url,
258
+ :method => @http_method,
259
+ :message => exception.message,
260
+ :class => exception.class.name,
261
+ :will_retry => will_retry
262
+ }
263
+ if @logger.debug?
264
+ # backtraces are big
265
+ log_params[:backtrace] = exception.backtrace
266
+ # headers can have sensitive data
267
+ log_params[:headers] = headers
268
+ # body can be big and may have sensitive data
269
+ log_params[:body] = body
290
270
  end
271
+ log_failure("Could not fetch URL", log_params)
291
272
 
292
- # Actually invoke the request in the background
293
- # Note: this must only be invoked after all handlers are defined, otherwise
294
- # those handlers are not guaranteed to be called!
295
- request.call
273
+ if will_retry
274
+ return :retry, event, attempt
275
+ else
276
+ return :failure, event, attempt
277
+ end
296
278
  end
297
279
 
298
280
  def close
@@ -301,24 +283,24 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
301
283
  end
302
284
 
303
285
  private
304
-
286
+
305
287
  def response_success?(response)
306
288
  code = response.code
307
289
  return true if @ignorable_codes && @ignorable_codes.include?(code)
308
290
  return code >= 200 && code <= 299
309
291
  end
310
-
292
+
311
293
  def retryable_response?(response)
312
- @retryable_codes.include?(response.code)
294
+ @retryable_codes && @retryable_codes.include?(response.code)
313
295
  end
314
-
296
+
315
297
  def retryable_exception?(exception)
316
298
  RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
317
299
  end
318
300
 
319
301
  # This is split into a separate method mostly to help testing
320
302
  def log_failure(message, opts)
321
- @logger.error("[HTTP Output Failure] #{message}", opts)
303
+ @logger.error(message, opts)
322
304
  end
323
305
 
324
306
  # Format the HTTP body
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-http'
3
- s.version = '5.2.2'
3
+ s.version = '5.3.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Sends events to a generic HTTP or HTTPS endpoint"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
20
20
 
21
21
  # Gem dependencies
22
22
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
23
- s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 7.0.0"
23
+ s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0", "< 8.0.0"
24
24
 
25
25
  s.add_development_dependency 'logstash-devutils'
26
26
  s.add_development_dependency 'sinatra'
@@ -3,6 +3,9 @@ require "logstash/outputs/http"
3
3
  require "logstash/codecs/plain"
4
4
  require "thread"
5
5
  require "sinatra"
6
+ require "webrick"
7
+ require "webrick/https"
8
+ require 'openssl'
6
9
  require_relative "../supports/compressed_requests"
7
10
 
8
11
  PORT = rand(65535-1024) + 1025
@@ -22,9 +25,20 @@ class TestApp < Sinatra::Base
22
25
  # on the fly uncompress gzip content
23
26
  use CompressedRequests
24
27
 
25
- # disable WEBrick logging
28
+ set :environment, :production
29
+ set :sessions, false
30
+
31
+ @@server_settings = {
32
+ :AccessLog => [], # disable WEBrick logging
33
+ :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL)
34
+ }
35
+
26
36
  def self.server_settings
27
- { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
37
+ @@server_settings
38
+ end
39
+
40
+ def self.server_settings=(settings)
41
+ @@server_settings = settings
28
42
  end
29
43
 
30
44
  def self.multiroute(methods, path, &block)
@@ -72,31 +86,22 @@ class TestApp < Sinatra::Base
72
86
  end
73
87
  end
74
88
 
75
- RSpec.configure do |config|
89
+ RSpec.configure do
76
90
  #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
77
- def sinatra_run_wait(app, opts)
91
+ def start_app_and_wait(app, opts = {})
78
92
  queue = Queue.new
79
93
 
80
- t = java.lang.Thread.new(
81
- proc do
82
- begin
83
- app.run!(opts) do |server|
84
- queue.push("started")
85
- end
86
- rescue => e
87
- puts "Error in webserver thread #{e}"
88
- # ignore
94
+ Thread.start do
95
+ begin
96
+ app.start!({ server: 'WEBrick', port: PORT }.merge opts) do |server|
97
+ queue.push(server)
89
98
  end
99
+ rescue => e
100
+ warn "Error starting app: #{e.inspect}" # ignore
90
101
  end
91
- )
92
- t.daemon = true
93
- t.start
94
- queue.pop # blocks until the run! callback runs
95
- end
102
+ end
96
103
 
97
- config.before(:suite) do
98
- sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
99
- puts "Test webserver on port #{PORT}"
104
+ queue.pop # blocks until the start! callback runs
100
105
  end
101
106
  end
102
107
 
@@ -104,6 +109,15 @@ describe LogStash::Outputs::Http do
104
109
  # Wait for the async request to finish in this spinlock
105
110
  # Requires pool_max to be 1
106
111
 
112
+ before(:all) do
113
+ @server = start_app_and_wait(TestApp)
114
+ end
115
+
116
+ after(:all) do
117
+ @server.shutdown # WEBrick::HTTPServer
118
+ TestApp.stop! rescue nil
119
+ end
120
+
107
121
  let(:port) { PORT }
108
122
  let(:event) {
109
123
  LogStash::Event.new({"message" => "hi"})
@@ -117,12 +131,10 @@ describe LogStash::Outputs::Http do
117
131
 
118
132
  let(:expected_method) { method.clone.to_sym }
119
133
  let(:client) { subject.client }
120
- let(:client_proxy) { subject.client.background }
121
134
 
122
135
  before do
123
- allow(client).to receive(:background).and_return(client_proxy)
124
136
  subject.register
125
- allow(client_proxy).to receive(:send).
137
+ allow(client).to receive(:send).
126
138
  with(expected_method, url, anything).
127
139
  and_call_original
128
140
  allow(subject).to receive(:log_failure).with(any_args)
@@ -142,7 +154,7 @@ describe LogStash::Outputs::Http do
142
154
  end
143
155
 
144
156
  it "should execute the request" do
145
- expect(client_proxy).to have_received(:send).
157
+ expect(client).to have_received(:send).
146
158
  with(expected_method, url, anything)
147
159
  end
148
160
  end
@@ -171,7 +183,7 @@ describe LogStash::Outputs::Http do
171
183
 
172
184
  context "with ignorable failing requests" do
173
185
  let(:url) { "http://localhost:#{port}/bad"}
174
- let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
186
+ let(:verb_behavior_config) { super().merge("ignorable_codes" => [400]) }
175
187
 
176
188
  before do
177
189
  subject.multi_receive([event])
@@ -199,7 +211,43 @@ describe LogStash::Outputs::Http do
199
211
  expect(subject).to have_received(:send_event).exactly(3).times
200
212
  end
201
213
  end
214
+ end
215
+
216
+ context "on exception" do
217
+ before :each do
218
+ allow(subject.client).to receive(:send).and_raise RuntimeError
219
+ subject.multi_receive([event])
220
+ end
221
+
222
+ it "should not log headers" do
223
+ expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
224
+ end
202
225
 
226
+ it "should not log the body" do
227
+ expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
228
+ end
229
+
230
+ context "with debug log level" do
231
+ before :all do
232
+ @current_log_level = LogStash::Logging::Logger.get_logging_context.get_root_logger.get_level.to_s.downcase
233
+ LogStash::Logging::Logger.configure_logging "debug"
234
+ end
235
+ after :all do
236
+ LogStash::Logging::Logger.configure_logging @current_log_level
237
+ end
238
+
239
+ it "should log a failure" do
240
+ expect(subject).to have_received(:log_failure).with(any_args)
241
+ end
242
+
243
+ it "should not log headers" do
244
+ expect(subject).to have_received(:log_failure).with(anything, hash_including(:headers))
245
+ end
246
+
247
+ it "should not log the body" do
248
+ expect(subject).to have_received(:log_failure).with(anything, hash_including(:body))
249
+ end
250
+ end
203
251
  end
204
252
  end
205
253
 
@@ -364,3 +412,75 @@ describe LogStash::Outputs::Http do
364
412
  end
365
413
  end
366
414
  end
415
+
416
+ describe LogStash::Outputs::Http do # different block as we're starting web server with TLS
417
+
418
+ @@default_server_settings = TestApp.server_settings.dup
419
+
420
+ before do
421
+ cert, key = WEBrick::Utils.create_self_signed_cert 2048, [["CN", ssl_cert_host]], "Logstash testing"
422
+ TestApp.server_settings = @@default_server_settings.merge({
423
+ :SSLEnable => true,
424
+ :SSLVerifyClient => OpenSSL::SSL::VERIFY_NONE,
425
+ :SSLCertificate => cert,
426
+ :SSLPrivateKey => key
427
+ })
428
+
429
+ TestApp.last_request = nil
430
+
431
+ @server = start_app_and_wait(TestApp)
432
+ end
433
+
434
+ after do
435
+ @server.shutdown # WEBrick::HTTPServer
436
+
437
+ TestApp.stop! rescue nil
438
+ TestApp.server_settings = @@default_server_settings
439
+ end
440
+
441
+ let(:ssl_cert_host) { 'localhost' }
442
+
443
+ let(:port) { PORT }
444
+ let(:url) { "https://localhost:#{port}/good" }
445
+ let(:method) { "post" }
446
+
447
+ let(:config) { { "url" => url, "http_method" => method } }
448
+
449
+ subject { LogStash::Outputs::Http.new(config) }
450
+
451
+ before { subject.register }
452
+ after { subject.close }
453
+
454
+ let(:last_request) { TestApp.last_request }
455
+ let(:last_request_body) { last_request.body.read }
456
+
457
+ let(:event) { LogStash::Event.new("message" => "hello!") }
458
+
459
+ context 'with default (full) verification' do
460
+
461
+ let(:config) { super() } # 'ssl_verification_mode' => 'full'
462
+
463
+ it "does NOT process the request (due client protocol exception)" do
464
+ # Manticore's default verification does not accept self-signed certificates!
465
+ Thread.start do
466
+ subject.multi_receive [ event ]
467
+ end
468
+ sleep 1.5
469
+
470
+ expect(last_request).to be nil
471
+ end
472
+
473
+ end
474
+
475
+ context 'with verification disabled' do
476
+
477
+ let(:config) { super().merge 'ssl_verification_mode' => 'none' }
478
+
479
+ it "should process the request" do
480
+ subject.multi_receive [ event ]
481
+ expect(last_request_body).to include '"message":"hello!"'
482
+ end
483
+
484
+ end
485
+
486
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.2.2
4
+ version: 5.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-06 00:00:00.000000000 Z
11
+ date: 2022-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -35,10 +35,10 @@ dependencies:
35
35
  requirements:
36
36
  - - ">="
37
37
  - !ruby/object:Gem::Version
38
- version: 6.0.0
38
+ version: 7.1.0
39
39
  - - "<"
40
40
  - !ruby/object:Gem::Version
41
- version: 7.0.0
41
+ version: 8.0.0
42
42
  name: logstash-mixin-http_client
43
43
  prerelease: false
44
44
  type: :runtime
@@ -46,10 +46,10 @@ dependencies:
46
46
  requirements:
47
47
  - - ">="
48
48
  - !ruby/object:Gem::Version
49
- version: 6.0.0
49
+ version: 7.1.0
50
50
  - - "<"
51
51
  - !ruby/object:Gem::Version
52
- version: 7.0.0
52
+ version: 8.0.0
53
53
  - !ruby/object:Gem::Dependency
54
54
  requirement: !ruby/object:Gem::Requirement
55
55
  requirements:
@@ -132,8 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
132
132
  - !ruby/object:Gem::Version
133
133
  version: '0'
134
134
  requirements: []
135
- rubyforge_project:
136
- rubygems_version: 2.6.13
135
+ rubygems_version: 3.1.6
137
136
  signing_key:
138
137
  specification_version: 4
139
138
  summary: Sends events to a generic HTTP or HTTPS endpoint