logstash-output-http 3.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 166795862fd592911c1662e2b5413de11f4824ba
4
- data.tar.gz: 028265896afc48b3e0e0c806007c523eec50ae71
3
+ metadata.gz: fb4ed46a005d9af73da085335c1f1f516cd54f1b
4
+ data.tar.gz: ea8ded7326eef70f098bbcab8e910d9e3cfe7b2a
5
5
  SHA512:
6
- metadata.gz: 1ca69a7fd82c6738025fd25eba6d2e3e728035e2e9d52bbf061fba6a4675392dc0d7744536546ade2b6564259806b42f80d7d7230a1755697548aea897d384fb
7
- data.tar.gz: 869ce7b82b48ed382b602c3188e425e45a771ce2a7274aba3dba2670bee009198d5879020cf5b3e46a58a9d1b9c2e61591136698732d6372b9a936ce05aa5c92
6
+ metadata.gz: 7c7e3a3e18f434a3d575f280cd5f8f6502833338a270519facf6d6f9115f78e033df68bcbfb6d050fcda56230a914cab22f61a1c06854203af51ce8d54e3dc03
7
+ data.tar.gz: 7e8b34fc4f8872a4754bc2868d48dbd57e7db5bee7006bfc2f6d6be116457e36a6d50edcb7a8fd8a3fb8747b7e2c0b022a387c8642f599e44bcc621dbf878a16
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 4.0.0
2
+ - Major overhaul of internals, adds new retry options
3
+ - Allow users to specify non-standard response codes as ignorable
4
+ - Set concurrency level to shared allowing for greater efficiency across threads
5
+
1
6
  ## 3.1.1
2
7
  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
3
8
 
@@ -19,6 +24,7 @@
19
24
  ## 2.1.1
20
25
  - Require http_client mixin with better keepalive handling
21
26
 
27
+
22
28
  ## 2.1.0
23
29
  - Properly close the client on #close
24
30
  - Optimized execution for Logstash 2.2 ng pipeline
@@ -39,4 +45,3 @@
39
45
  - Concurrent execution
40
46
  - Add many HTTP options via the http_client mixin
41
47
  - Switch to manticore as HTTP Client
42
-
@@ -7,8 +7,18 @@ require "logstash/plugin_mixins/http_client"
7
7
 
8
8
  class LogStash::Outputs::Http < LogStash::Outputs::Base
9
9
  include LogStash::PluginMixins::HttpClient
10
+
11
+ concurrency :shared
10
12
 
11
13
  VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
14
+
15
+ RETRYABLE_MANTICORE_EXCEPTIONS = [
16
+ ::Manticore::Timeout,
17
+ ::Manticore::SocketException,
18
+ ::Manticore::ClientProtocolException,
19
+ ::Manticore::ResolutionFailure,
20
+ ::Manticore::SocketTimeout
21
+ ]
12
22
 
13
23
  # This output lets you send events to a
14
24
  # generic HTTP(S) endpoint
@@ -40,13 +50,24 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
40
50
  # * if format is "json", "application/json"
41
51
  # * if format is "form", "application/x-www-form-urlencoded"
42
52
  config :content_type, :validate => :string
53
+
54
+ # Set this to false if you don't want this output to retry failed requests
55
+ config :retry_failed, :validate => :boolean, :default => true
56
+
57
+ # If encountered as response codes this plugin will retry these requests
58
+ config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
59
+
60
+ # If you would like to consider some non-2xx codes to be successes
61
+ # enumerate them here. Responses returning these codes will be considered successes
62
+ config :ignorable_codes, :validate => :number, :list => true
43
63
 
44
64
  # This lets you choose the structure and parts of the event that are sent.
45
65
  #
46
66
  #
47
67
  # For example:
48
68
  # [source,ruby]
49
- # mapping => {"foo", "%{host}", "bar", "%{type}"}
69
+ # mapping => {"foo" => "%{host}"
70
+ # "bar" => "%{type}"}
50
71
  config :mapping, :validate => :hash
51
72
 
52
73
  # Set the format of the http body.
@@ -82,70 +103,181 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
82
103
  end
83
104
 
84
105
  validate_format!
106
+
107
+ # Run named Timer as daemon thread
108
+ @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
85
109
  end # def register
86
110
 
87
111
  def multi_receive(events)
88
- events.each {|event| receive(event, :parallel)}
89
- client.execute!
112
+ send_events(events)
90
113
  end
91
-
92
- # Once we no longer need to support Logstash < 2.2 (pre-ng-pipeline)
93
- # We don't need to handle :background style requests
94
- #
95
- # We use :background style requests for Logstash < 2.2 because before the microbatching
96
- # pipeline performance is greatly improved by having some degree of async behavior.
97
- #
98
- # In Logstash 2.2 and after things are much simpler, we just run each batch in parallel
99
- # This will make performance much easier to reason about, and more importantly let us guarantee
100
- # that if `multi_receive` returns all items have been sent.
101
- def receive(event, async_type=:background)
114
+
115
+ class RetryTimerTask < java.util.TimerTask
116
+ def initialize(pending, event, attempt)
117
+ @pending = pending
118
+ @event = event
119
+ @attempt = attempt
120
+ super()
121
+ end
122
+
123
+ def run
124
+ @pending << [@event, @attempt]
125
+ end
126
+ end
127
+
128
+ def send_events(events)
129
+ successes = java.util.concurrent.atomic.AtomicInteger.new(0)
130
+ failures = java.util.concurrent.atomic.AtomicInteger.new(0)
131
+ retries = java.util.concurrent.atomic.AtomicInteger.new(0)
132
+
133
+ pending = Queue.new
134
+ events.each {|e| pending << [e, 0]}
135
+
136
+ while popped = pending.pop
137
+ break if popped == :done
138
+
139
+ event, attempt = popped
140
+
141
+ send_event(event, attempt) do |action,event,attempt|
142
+ begin
143
+ action = :failure if action == :retry && !@retry_failed
144
+
145
+ case action
146
+ when :success
147
+ successes.incrementAndGet
148
+ when :retry
149
+ retries.incrementAndGet
150
+
151
+ next_attempt = attempt+1
152
+ sleep_for = sleep_for_attempt(next_attempt)
153
+ @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
154
+ timer_task = RetryTimerTask.new(pending, event, next_attempt)
155
+ @timer.schedule(timer_task, sleep_for*1000)
156
+ when :failure
157
+ failures.incrementAndGet
158
+ else
159
+ raise "Unknown action #{action}"
160
+ end
161
+
162
+ if action == :success || action == :failure
163
+ if successes.get+failures.get == events.size
164
+ pending << :done
165
+ end
166
+ end
167
+ rescue => e
168
+ # This should never happen unless there's a flat out bug in the code
169
+ @logger.error("Error sending HTTP Request",
170
+ :class => e.class.name,
171
+ :message => e.message,
172
+ :backtrace => e.backtrace)
173
+ failures.incrementAndGet
174
+ raise e
175
+ end
176
+ end
177
+ end
178
+ rescue => e
179
+ @logger.error("Error in http output loop",
180
+ :class => e.class.name,
181
+ :message => e.message,
182
+ :backtrace => e.backtrace)
183
+ raise e
184
+ end
185
+
186
+ def sleep_for_attempt(attempt)
187
+ sleep_for = attempt**2
188
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
189
+ (sleep_for/2) + (rand(0..sleep_for)/2)
190
+ end
191
+
192
+ def send_event(event, attempt)
102
193
  body = event_body(event)
103
194
 
104
- # Block waiting for a token
105
- token = @request_tokens.pop if async_type == :background
106
-
107
195
  # Send the request
108
196
  url = event.sprintf(@url)
109
197
  headers = event_headers(event)
110
198
 
111
199
  # Create an async request
112
- request = client.send(async_type).send(@http_method, url, :body => body, :headers => headers)
113
-
114
- request.on_complete do
115
- # Make sure we return the token to the pool
116
- @request_tokens << token if async_type == :background
117
- end
200
+ request = client.background.send(@http_method, url, :body => body, :headers => headers)
201
+ request.call # Actually invoke the request in the background
118
202
 
119
203
  request.on_success do |response|
120
- if response.code < 200 || response.code > 299
121
- log_failure(
122
- "Encountered non-200 HTTP code #{200}",
123
- :response_code => response.code,
124
- :url => url,
125
- :event => event)
204
+ begin
205
+ if !response_success?(response)
206
+ will_retry = retryable_response?(response)
207
+ log_failure(
208
+ "Encountered non-2xx HTTP code #{response.code}",
209
+ :response_code => response.code,
210
+ :url => url,
211
+ :event => event,
212
+ :will_retry => will_retry
213
+ )
214
+
215
+ if will_retry
216
+ yield :retry, event, attempt
217
+ else
218
+ yield :failure, event, attempt
219
+ end
220
+ else
221
+ yield :success, event, attempt
222
+ end
223
+ rescue => e
224
+ # Shouldn't ever happen
225
+ @logger.error("Unexpected error in request success!",
226
+ :class => e.class.name,
227
+ :message => e.message,
228
+ :backtrace => e.backtrace)
126
229
  end
127
230
  end
128
231
 
129
232
  request.on_failure do |exception|
130
- log_failure("Could not fetch URL",
131
- :url => url,
132
- :method => @http_method,
133
- :body => body,
134
- :headers => headers,
135
- :message => exception.message,
136
- :class => exception.class.name,
137
- :backtrace => exception.backtrace
138
- )
233
+ begin
234
+ will_retry = retryable_exception?(exception)
235
+ log_failure("Could not fetch URL",
236
+ :url => url,
237
+ :method => @http_method,
238
+ :body => body,
239
+ :headers => headers,
240
+ :message => exception.message,
241
+ :class => exception.class.name,
242
+ :backtrace => exception.backtrace,
243
+ :will_retry => will_retry
244
+ )
245
+
246
+ if will_retry
247
+ yield :retry, event, attempt
248
+ else
249
+ yield :failure, event, attempt
250
+ end
251
+ rescue => e
252
+ # Shouldn't ever happen
253
+ @logger.error("Unexpected error in request failure!",
254
+ :class => e.class.name,
255
+ :message => e.message,
256
+ :backtrace => e.backtrace)
257
+ end
139
258
  end
140
-
141
- request.call if async_type == :background
142
259
  end
143
260
 
144
261
  def close
262
+ @timer.cancel
145
263
  client.close
146
264
  end
147
265
 
148
266
  private
267
+
268
+ def response_success?(response)
269
+ code = response.code
270
+ return true if @ignorable_codes && @ignorable_codes.include?(code)
271
+ return code >= 200 && code <= 299
272
+ end
273
+
274
+ def retryable_response?(response)
275
+ @retryable_codes.include?(response.code)
276
+ end
277
+
278
+ def retryable_exception?(exception)
279
+ RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
280
+ end
149
281
 
150
282
  # This is split into a separate method mostly to help testing
151
283
  def log_failure(message, opts)
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-http'
4
- s.version = '3.1.1'
4
+ s.version = '4.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -27,4 +27,3 @@ Gem::Specification.new do |s|
27
27
  s.add_development_dependency 'sinatra'
28
28
  s.add_development_dependency 'webrick'
29
29
  end
30
-
@@ -1,5 +1,6 @@
1
1
  require "logstash/devutils/rspec/spec_helper"
2
2
  require "logstash/outputs/http"
3
+ require "logstash/codecs/plain"
3
4
  require "thread"
4
5
  require "sinatra"
5
6
 
@@ -37,6 +38,14 @@ class TestApp < Sinatra::Base
37
38
  def self.last_request
38
39
  @last_request
39
40
  end
41
+
42
+ def self.retry_fail_count=(count)
43
+ @retry_fail_count = count
44
+ end
45
+
46
+ def self.retry_fail_count()
47
+ @retry_fail_count
48
+ end
40
49
 
41
50
  multiroute(%w(get post put patch delete), "/good") do
42
51
  self.class.last_request = request
@@ -45,7 +54,18 @@ class TestApp < Sinatra::Base
45
54
 
46
55
  multiroute(%w(get post put patch delete), "/bad") do
47
56
  self.class.last_request = request
48
- [500, "YUP"]
57
+ [400, "YUP"]
58
+ end
59
+
60
+ multiroute(%w(get post put patch delete), "/retry") do
61
+ self.class.last_request = request
62
+
63
+ if self.class.retry_fail_count > 0
64
+ self.class.retry_fail_count -= 1
65
+ [429, "Will succeed in #{self.class.retry_fail_count}"]
66
+ else
67
+ [200, "Done Retrying"]
68
+ end
49
69
  end
50
70
  end
51
71
 
@@ -54,80 +74,48 @@ RSpec.configure do |config|
54
74
  def sinatra_run_wait(app, opts)
55
75
  queue = Queue.new
56
76
 
57
- Thread.new(queue) do |queue|
58
- begin
59
- app.run!(opts) do |server|
60
- queue.push("started")
77
+ t = java.lang.Thread.new(
78
+ proc do
79
+ begin
80
+ app.run!(opts) do |server|
81
+ queue.push("started")
82
+ end
83
+ rescue => e
84
+ puts "Error in webserver thread #{e}"
85
+ # ignore
61
86
  end
62
- rescue
63
- # ignore
64
87
  end
65
- end
66
-
88
+ )
89
+ t.daemon = true
90
+ t.start
67
91
  queue.pop # blocks until the run! callback runs
68
92
  end
69
93
 
70
94
  config.before(:suite) do
71
95
  sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
96
+ puts "Test webserver on port #{PORT}"
72
97
  end
73
98
  end
74
99
 
75
100
  describe LogStash::Outputs::Http do
76
101
  # Wait for the async request to finish in this spinlock
77
102
  # Requires pool_max to be 1
78
- def wait_for_request
79
-
80
- loop do
81
- sleep(0.1)
82
- break if subject.request_tokens.size > 0
83
- end
84
- end
85
103
 
86
104
  let(:port) { PORT }
87
105
  let(:event) { LogStash::Event.new("message" => "hi") }
88
106
  let(:url) { "http://localhost:#{port}/good" }
89
107
  let(:method) { "post" }
90
108
 
91
- describe "when num requests > token count" do
92
- let(:pool_max) { 10 }
93
- let(:num_reqs) { pool_max / 2 }
94
- let(:client) { subject.client }
95
- let(:client_proxy) { subject.client.background }
96
-
97
- subject {
98
- LogStash::Outputs::Http.new("url" => url,
99
- "http_method" => method,
100
- "pool_max" => pool_max)
101
- }
102
-
103
- before do
104
- allow(client).to receive(:background).and_return(client_proxy)
105
- subject.register
106
- end
107
-
108
- after do
109
- subject.close
110
- end
111
-
112
- it "should receive all the requests" do
113
- expect(client_proxy).to receive(:send).
114
- with(method.to_sym, url, anything).
115
- exactly(num_reqs).times.
116
- and_call_original
117
-
118
- num_reqs.times {|t| subject.receive(event)}
119
- end
120
- end
121
-
122
- shared_examples("verb behavior") do |method, async_type|
123
- subject { LogStash::Outputs::Http.new("url" => url, "http_method" => method, "pool_max" => 1) }
109
+ shared_examples("verb behavior") do |method|
110
+ let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
111
+ subject { LogStash::Outputs::Http.new(verb_behavior_config) }
124
112
 
125
113
  let(:expected_method) { method.clone.to_sym }
126
114
  let(:client) { subject.client }
127
- let(:client_proxy) { subject.client.send(async_type) }
115
+ let(:client_proxy) { subject.client.background }
128
116
 
129
117
  before do
130
- allow(client).to receive(async_type).and_return(client_proxy)
118
+ allow(client).to receive(:background).and_return(client_proxy)
131
119
  subject.register
132
120
  allow(client_proxy).to receive(:send).
133
121
  with(expected_method, url, anything).
@@ -138,7 +126,7 @@ describe LogStash::Outputs::Http do
138
126
  context "performing a get" do
139
127
  describe "invoking the request" do
140
128
  before do
141
- subject.receive(event, async_type)
129
+ subject.multi_receive([event])
142
130
  end
143
131
 
144
132
  it "should execute the request" do
@@ -149,7 +137,7 @@ describe LogStash::Outputs::Http do
149
137
 
150
138
  context "with passing requests" do
151
139
  before do
152
- subject.receive(event)
140
+ subject.multi_receive([event])
153
141
  end
154
142
 
155
143
  it "should not log a failure" do
@@ -161,29 +149,51 @@ describe LogStash::Outputs::Http do
161
149
  let(:url) { "http://localhost:#{port}/bad"}
162
150
 
163
151
  before do
164
- subject.receive(event, async_type)
165
-
166
- if async_type == :background
167
- wait_for_request
168
- else
169
- subject.client.execute!
170
- end
152
+ subject.multi_receive([event])
171
153
  end
172
154
 
173
155
  it "should log a failure" do
174
156
  expect(subject).to have_received(:log_failure).with(any_args)
175
157
  end
176
158
  end
159
+
160
+ context "with ignorable failing requests" do
161
+ let(:url) { "http://localhost:#{port}/bad"}
162
+ let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
163
+
164
+ before do
165
+ subject.multi_receive([event])
166
+ end
167
+
168
+ it "should log a failure" do
169
+ expect(subject).not_to have_received(:log_failure).with(any_args)
170
+ end
171
+ end
172
+
173
+ context "with retryable failing requests" do
174
+ let(:url) { "http://localhost:#{port}/retry"}
175
+
176
+ before do
177
+ TestApp.retry_fail_count=2
178
+ allow(subject).to receive(:send_event).and_call_original
179
+ subject.multi_receive([event])
180
+ end
181
+
182
+ it "should log a failure 2 times" do
183
+ expect(subject).to have_received(:log_failure).with(any_args).twice
184
+ end
185
+
186
+ it "should make three total requests" do
187
+ expect(subject).to have_received(:send_event).exactly(3).times
188
+ end
189
+ end
190
+
177
191
  end
178
192
  end
179
193
 
180
194
  LogStash::Outputs::Http::VALID_METHODS.each do |method|
181
- context "when using '#{method}' via :background" do
182
- include_examples("verb behavior", method, :background)
183
- end
184
-
185
- context "when using '#{method}' via :parallel" do
186
- include_examples("verb behavior", method, :parallel)
195
+ context "when using '#{method}'" do
196
+ include_examples("verb behavior", method)
187
197
  end
188
198
  end
189
199
 
@@ -193,8 +203,7 @@ describe LogStash::Outputs::Http do
193
203
  end
194
204
 
195
205
  before do
196
- subject.receive(event)
197
- wait_for_request
206
+ subject.multi_receive([event])
198
207
  end
199
208
 
200
209
  let(:last_request) { TestApp.last_request }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.1
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-14 00:00:00.000000000 Z
11
+ date: 2017-02-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -129,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
129
129
  version: '0'
130
130
  requirements: []
131
131
  rubyforge_project:
132
- rubygems_version: 2.6.3
132
+ rubygems_version: 2.4.8
133
133
  signing_key:
134
134
  specification_version: 4
135
135
  summary: This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint