logstash-output-http 5.2.2 → 5.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/http.rb +80 -105
- data/logstash-output-http.gemspec +1 -1
- data/spec/outputs/http_spec.rb +2 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4296b53881ec5d3e420667e9aa554a25781397858a38cbbd9a1d41b6ff49f91b
|
4
|
+
data.tar.gz: 07c7e8e0b029d23c18855f86147bef1848f6eac1625afe86344e7d24b6dc4a6f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cccd68833da4cb41c5699541a7c67065e4b6c0d476f1076dbf59f103912050689756a20bb3d4b53468a08bf84e2c088a3fdf5586c5f19b6032504d3ba3cc698d
|
7
|
+
data.tar.gz: bcc121d030f1f2bcaa5de4f7a498610338b35a8df01d2262a7d28cc46472780896d56551aa058cfec5dbd8f12e132bd1ac67321a36c6cb1bb474faaf1b604f34
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 5.2.3
|
2
|
+
- Fixed handling of empty `retryable_codes` [#99](https://github.com/logstash-plugins/logstash-output-http/pull/99)
|
3
|
+
|
1
4
|
## 5.2.2
|
2
5
|
- Fix high CPU usage on retries in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
|
3
6
|
- Enable compression in json_batch mode. [#89](https://github.com/logstash-plugins/logstash-output-http/pull/89)
|
@@ -8,18 +8,18 @@ require "zlib"
|
|
8
8
|
|
9
9
|
class LogStash::Outputs::Http < LogStash::Outputs::Base
|
10
10
|
include LogStash::PluginMixins::HttpClient
|
11
|
-
|
11
|
+
|
12
12
|
concurrency :shared
|
13
13
|
|
14
14
|
attr_accessor :is_batch
|
15
15
|
|
16
16
|
VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
|
17
|
-
|
17
|
+
|
18
18
|
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
19
19
|
::Manticore::Timeout,
|
20
20
|
::Manticore::SocketException,
|
21
|
-
::Manticore::ClientProtocolException,
|
22
|
-
::Manticore::ResolutionFailure,
|
21
|
+
::Manticore::ClientProtocolException,
|
22
|
+
::Manticore::ResolutionFailure,
|
23
23
|
::Manticore::SocketTimeout
|
24
24
|
]
|
25
25
|
|
@@ -53,14 +53,14 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
53
53
|
# * if format is "json", "application/json"
|
54
54
|
# * if format is "form", "application/x-www-form-urlencoded"
|
55
55
|
config :content_type, :validate => :string
|
56
|
-
|
56
|
+
|
57
57
|
# Set this to false if you don't want this output to retry failed requests
|
58
58
|
config :retry_failed, :validate => :boolean, :default => true
|
59
|
-
|
59
|
+
|
60
60
|
# If encountered as response codes this plugin will retry these requests
|
61
61
|
config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
|
62
|
-
|
63
|
-
# If you would like to consider some non-2xx codes to be successes
|
62
|
+
|
63
|
+
# If you would like to consider some non-2xx codes to be successes
|
64
64
|
# enumerate them here. Responses returning these codes will be considered successes
|
65
65
|
config :ignorable_codes, :validate => :number, :list => true
|
66
66
|
|
@@ -85,7 +85,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
85
85
|
|
86
86
|
# Set this to true if you want to enable gzip compression for your http requests
|
87
87
|
config :http_compression, :validate => :boolean, :default => false
|
88
|
-
|
88
|
+
|
89
89
|
config :message, :validate => :string
|
90
90
|
|
91
91
|
def register
|
@@ -114,7 +114,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
114
114
|
@headers["Content-Type"] = @content_type
|
115
115
|
|
116
116
|
validate_format!
|
117
|
-
|
117
|
+
|
118
118
|
# Run named Timer as daemon thread
|
119
119
|
@timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
|
120
120
|
end # def register
|
@@ -123,7 +123,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
123
123
|
return if events.empty?
|
124
124
|
send_events(events)
|
125
125
|
end
|
126
|
-
|
126
|
+
|
127
127
|
class RetryTimerTask < java.util.TimerTask
|
128
128
|
def initialize(pending, event, attempt)
|
129
129
|
@pending = pending
|
@@ -131,7 +131,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
131
131
|
@attempt = attempt
|
132
132
|
super()
|
133
133
|
end
|
134
|
-
|
134
|
+
|
135
135
|
def run
|
136
136
|
@pending << [@event, @attempt]
|
137
137
|
end
|
@@ -153,7 +153,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
153
153
|
:event => event
|
154
154
|
)
|
155
155
|
end
|
156
|
-
|
156
|
+
|
157
157
|
def send_events(events)
|
158
158
|
successes = java.util.concurrent.atomic.AtomicInteger.new(0)
|
159
159
|
failures = java.util.concurrent.atomic.AtomicInteger.new(0)
|
@@ -169,44 +169,43 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
169
169
|
|
170
170
|
while popped = pending.pop
|
171
171
|
break if popped == :done
|
172
|
-
|
172
|
+
|
173
173
|
event, attempt = popped
|
174
174
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
failures.incrementAndGet
|
192
|
-
else
|
193
|
-
raise "Unknown action #{action}"
|
194
|
-
end
|
195
|
-
|
196
|
-
if action == :success || action == :failure
|
197
|
-
if successes.get+failures.get == event_count
|
198
|
-
pending << :done
|
199
|
-
end
|
200
|
-
end
|
201
|
-
rescue => e
|
202
|
-
# This should never happen unless there's a flat out bug in the code
|
203
|
-
@logger.error("Error sending HTTP Request",
|
204
|
-
:class => e.class.name,
|
205
|
-
:message => e.message,
|
206
|
-
:backtrace => e.backtrace)
|
175
|
+
action, event, attempt = send_event(event, attempt)
|
176
|
+
begin
|
177
|
+
action = :failure if action == :retry && !@retry_failed
|
178
|
+
|
179
|
+
case action
|
180
|
+
when :success
|
181
|
+
successes.incrementAndGet
|
182
|
+
when :retry
|
183
|
+
retries.incrementAndGet
|
184
|
+
|
185
|
+
next_attempt = attempt+1
|
186
|
+
sleep_for = sleep_for_attempt(next_attempt)
|
187
|
+
@logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
|
188
|
+
timer_task = RetryTimerTask.new(pending, event, next_attempt)
|
189
|
+
@timer.schedule(timer_task, sleep_for*1000)
|
190
|
+
when :failure
|
207
191
|
failures.incrementAndGet
|
208
|
-
|
192
|
+
else
|
193
|
+
raise "Unknown action #{action}"
|
209
194
|
end
|
195
|
+
|
196
|
+
if action == :success || action == :failure
|
197
|
+
if successes.get+failures.get == event_count
|
198
|
+
pending << :done
|
199
|
+
end
|
200
|
+
end
|
201
|
+
rescue => e
|
202
|
+
# This should never happen unless there's a flat out bug in the code
|
203
|
+
@logger.error("Error sending HTTP Request",
|
204
|
+
:class => e.class.name,
|
205
|
+
:message => e.message,
|
206
|
+
:backtrace => e.backtrace)
|
207
|
+
failures.incrementAndGet
|
208
|
+
raise e
|
210
209
|
end
|
211
210
|
end
|
212
211
|
rescue => e
|
@@ -216,13 +215,13 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
216
215
|
:backtrace => e.backtrace)
|
217
216
|
raise e
|
218
217
|
end
|
219
|
-
|
218
|
+
|
220
219
|
def sleep_for_attempt(attempt)
|
221
220
|
sleep_for = attempt**2
|
222
221
|
sleep_for = sleep_for <= 60 ? sleep_for : 60
|
223
222
|
(sleep_for/2) + (rand(0..sleep_for)/2)
|
224
223
|
end
|
225
|
-
|
224
|
+
|
226
225
|
def send_event(event, attempt)
|
227
226
|
body = event_body(event)
|
228
227
|
|
@@ -237,62 +236,38 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
237
236
|
end
|
238
237
|
|
239
238
|
# Create an async request
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
log_error_response(response, url, event)
|
250
|
-
yield :failure, event, attempt
|
251
|
-
end
|
252
|
-
else
|
253
|
-
yield :success, event, attempt
|
254
|
-
end
|
255
|
-
rescue => e
|
256
|
-
# Shouldn't ever happen
|
257
|
-
@logger.error("Unexpected error in request success!",
|
258
|
-
:class => e.class.name,
|
259
|
-
:message => e.message,
|
260
|
-
:backtrace => e.backtrace)
|
239
|
+
response = client.send(@http_method, url, :body => body, :headers => headers).call
|
240
|
+
|
241
|
+
if !response_success?(response)
|
242
|
+
if retryable_response?(response)
|
243
|
+
log_retryable_response(response)
|
244
|
+
return :retry, event, attempt
|
245
|
+
else
|
246
|
+
log_error_response(response, url, event)
|
247
|
+
return :failure, event, attempt
|
261
248
|
end
|
249
|
+
else
|
250
|
+
return :success, event, attempt
|
262
251
|
end
|
263
252
|
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
yield :failure, event, attempt
|
282
|
-
end
|
283
|
-
rescue => e
|
284
|
-
# Shouldn't ever happen
|
285
|
-
@logger.error("Unexpected error in request failure!",
|
286
|
-
:class => e.class.name,
|
287
|
-
:message => e.message,
|
288
|
-
:backtrace => e.backtrace)
|
289
|
-
end
|
253
|
+
rescue => exception
|
254
|
+
will_retry = retryable_exception?(exception)
|
255
|
+
log_failure("Could not fetch URL",
|
256
|
+
:url => url,
|
257
|
+
:method => @http_method,
|
258
|
+
:body => body,
|
259
|
+
:headers => headers,
|
260
|
+
:message => exception.message,
|
261
|
+
:class => exception.class.name,
|
262
|
+
:backtrace => exception.backtrace,
|
263
|
+
:will_retry => will_retry
|
264
|
+
)
|
265
|
+
|
266
|
+
if will_retry
|
267
|
+
return :retry, event, attempt
|
268
|
+
else
|
269
|
+
return :failure, event, attempt
|
290
270
|
end
|
291
|
-
|
292
|
-
# Actually invoke the request in the background
|
293
|
-
# Note: this must only be invoked after all handlers are defined, otherwise
|
294
|
-
# those handlers are not guaranteed to be called!
|
295
|
-
request.call
|
296
271
|
end
|
297
272
|
|
298
273
|
def close
|
@@ -301,17 +276,17 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
|
|
301
276
|
end
|
302
277
|
|
303
278
|
private
|
304
|
-
|
279
|
+
|
305
280
|
def response_success?(response)
|
306
281
|
code = response.code
|
307
282
|
return true if @ignorable_codes && @ignorable_codes.include?(code)
|
308
283
|
return code >= 200 && code <= 299
|
309
284
|
end
|
310
|
-
|
285
|
+
|
311
286
|
def retryable_response?(response)
|
312
|
-
@retryable_codes.include?(response.code)
|
287
|
+
@retryable_codes && @retryable_codes.include?(response.code)
|
313
288
|
end
|
314
|
-
|
289
|
+
|
315
290
|
def retryable_exception?(exception)
|
316
291
|
RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
|
317
292
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-http'
|
3
|
-
s.version = '5.2.
|
3
|
+
s.version = '5.2.3'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Sends events to a generic HTTP or HTTPS endpoint"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/outputs/http_spec.rb
CHANGED
@@ -117,12 +117,10 @@ describe LogStash::Outputs::Http do
|
|
117
117
|
|
118
118
|
let(:expected_method) { method.clone.to_sym }
|
119
119
|
let(:client) { subject.client }
|
120
|
-
let(:client_proxy) { subject.client.background }
|
121
120
|
|
122
121
|
before do
|
123
|
-
allow(client).to receive(:background).and_return(client_proxy)
|
124
122
|
subject.register
|
125
|
-
allow(
|
123
|
+
allow(client).to receive(:send).
|
126
124
|
with(expected_method, url, anything).
|
127
125
|
and_call_original
|
128
126
|
allow(subject).to receive(:log_failure).with(any_args)
|
@@ -142,7 +140,7 @@ describe LogStash::Outputs::Http do
|
|
142
140
|
end
|
143
141
|
|
144
142
|
it "should execute the request" do
|
145
|
-
expect(
|
143
|
+
expect(client).to have_received(:send).
|
146
144
|
with(expected_method, url, anything)
|
147
145
|
end
|
148
146
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.2.
|
4
|
+
version: 5.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|