logstash-output-bcdb 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,364 +1,364 @@
1
- require "logstash/devutils/rspec/spec_helper"
2
- require "logstash/outputs/http"
3
- require "logstash/codecs/plain"
4
- require "thread"
5
- require "sinatra"
6
- require_relative "../supports/compressed_requests"
7
-
8
- PORT = rand(65535-1024) + 1025
9
-
10
- class LogStash::Outputs::Http
11
- attr_writer :agent
12
- attr_reader :request_tokens
13
- end
14
-
15
- # note that Sinatra startup and shutdown messages are directly logged to stderr so
16
- # it is not really possible to disable them without reopening stderr which is not advisable.
17
- #
18
- # == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
19
- # == Sinatra has ended his set (crowd applauds)
20
- #
21
- class TestApp < Sinatra::Base
22
- # on the fly uncompress gzip content
23
- use CompressedRequests
24
-
25
- # disable WEBrick logging
26
- def self.server_settings
27
- { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
28
- end
29
-
30
- def self.multiroute(methods, path, &block)
31
- methods.each do |method|
32
- method.to_sym
33
- self.send method, path, &block
34
- end
35
- end
36
-
37
- def self.last_request=(request)
38
- @last_request = request
39
- end
40
-
41
- def self.last_request
42
- @last_request
43
- end
44
-
45
- def self.retry_fail_count=(count)
46
- @retry_fail_count = count
47
- end
48
-
49
- def self.retry_fail_count()
50
- @retry_fail_count || 2
51
- end
52
-
53
- multiroute(%w(get post put patch delete), "/good") do
54
- self.class.last_request = request
55
- [200, "YUP"]
56
- end
57
-
58
- multiroute(%w(get post put patch delete), "/bad") do
59
- self.class.last_request = request
60
- [400, "YUP"]
61
- end
62
-
63
- multiroute(%w(get post put patch delete), "/retry") do
64
- self.class.last_request = request
65
-
66
- if self.class.retry_fail_count > 0
67
- self.class.retry_fail_count -= 1
68
- [429, "Will succeed in #{self.class.retry_fail_count}"]
69
- else
70
- [200, "Done Retrying"]
71
- end
72
- end
73
- end
74
-
75
- RSpec.configure do |config|
76
- #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
77
- def sinatra_run_wait(app, opts)
78
- queue = Queue.new
79
-
80
- t = java.lang.Thread.new(
81
- proc do
82
- begin
83
- app.run!(opts) do |server|
84
- queue.push("started")
85
- end
86
- rescue => e
87
- puts "Error in webserver thread #{e}"
88
- # ignore
89
- end
90
- end
91
- )
92
- t.daemon = true
93
- t.start
94
- queue.pop # blocks until the run! callback runs
95
- end
96
-
97
- config.before(:suite) do
98
- sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
99
- puts "Test webserver on port #{PORT}"
100
- end
101
- end
102
-
103
- describe LogStash::Outputs::Http do
104
- # Wait for the async request to finish in this spinlock
105
- # Requires pool_max to be 1
106
-
107
- let(:port) { PORT }
108
- let(:event) {
109
- LogStash::Event.new({"message" => "hi"})
110
- }
111
- let(:url) { "http://localhost:#{port}/good" }
112
- let(:method) { "post" }
113
-
114
- shared_examples("verb behavior") do |method|
115
- let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
116
- subject { LogStash::Outputs::Http.new(verb_behavior_config) }
117
-
118
- let(:expected_method) { method.clone.to_sym }
119
- let(:client) { subject.client }
120
-
121
- before do
122
- subject.register
123
- allow(client).to receive(:send).
124
- with(expected_method, url, anything).
125
- and_call_original
126
- allow(subject).to receive(:log_failure).with(any_args)
127
- allow(subject).to receive(:log_retryable_response).with(any_args)
128
- end
129
-
130
- context 'sending no events' do
131
- it 'should not block the pipeline' do
132
- subject.multi_receive([])
133
- end
134
- end
135
-
136
- context "performing a get" do
137
- describe "invoking the request" do
138
- before do
139
- subject.multi_receive([event])
140
- end
141
-
142
- it "should execute the request" do
143
- expect(client).to have_received(:send).
144
- with(expected_method, url, anything)
145
- end
146
- end
147
-
148
- context "with passing requests" do
149
- before do
150
- subject.multi_receive([event])
151
- end
152
-
153
- it "should not log a failure" do
154
- expect(subject).not_to have_received(:log_failure).with(any_args)
155
- end
156
- end
157
-
158
- context "with failing requests" do
159
- let(:url) { "http://localhost:#{port}/bad"}
160
-
161
- before do
162
- subject.multi_receive([event])
163
- end
164
-
165
- it "should log a failure" do
166
- expect(subject).to have_received(:log_failure).with(any_args)
167
- end
168
- end
169
-
170
- context "with ignorable failing requests" do
171
- let(:url) { "http://localhost:#{port}/bad"}
172
- let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
173
-
174
- before do
175
- subject.multi_receive([event])
176
- end
177
-
178
- it "should log a failure" do
179
- expect(subject).not_to have_received(:log_failure).with(any_args)
180
- end
181
- end
182
-
183
- context "with retryable failing requests" do
184
- let(:url) { "http://localhost:#{port}/retry"}
185
-
186
- before do
187
- TestApp.retry_fail_count=2
188
- allow(subject).to receive(:send_event).and_call_original
189
- subject.multi_receive([event])
190
- end
191
-
192
- it "should log a retryable response 2 times" do
193
- expect(subject).to have_received(:log_retryable_response).with(any_args).twice
194
- end
195
-
196
- it "should make three total requests" do
197
- expect(subject).to have_received(:send_event).exactly(3).times
198
- end
199
- end
200
-
201
- end
202
- end
203
-
204
- LogStash::Outputs::Http::VALID_METHODS.each do |method|
205
- context "when using '#{method}'" do
206
- include_examples("verb behavior", method)
207
- end
208
- end
209
-
210
- shared_examples("a received event") do
211
- before do
212
- TestApp.last_request = nil
213
- end
214
-
215
- let(:events) { [event] }
216
-
217
- describe "with a good code" do
218
- before do
219
- subject.multi_receive(events)
220
- end
221
-
222
- let(:last_request) { TestApp.last_request }
223
- let(:body) { last_request.body.read }
224
- let(:content_type) { last_request.env["CONTENT_TYPE"] }
225
-
226
- it "should receive the request" do
227
- expect(last_request).to be_truthy
228
- end
229
-
230
- it "should receive the event as a hash" do
231
- expect(body).to eql(expected_body)
232
- end
233
-
234
- it "should have the correct content type" do
235
- expect(content_type).to eql(expected_content_type)
236
- end
237
- end
238
-
239
- describe "a retryable code" do
240
- let(:url) { "http://localhost:#{port}/retry" }
241
-
242
- before do
243
- TestApp.retry_fail_count=2
244
- allow(subject).to receive(:send_event).and_call_original
245
- allow(subject).to receive(:log_retryable_response)
246
- subject.multi_receive(events)
247
- end
248
-
249
- it "should retry" do
250
- expect(subject).to have_received(:log_retryable_response).with(any_args).twice
251
- end
252
- end
253
- end
254
-
255
- shared_examples "integration tests" do
256
- let(:base_config) { {} }
257
- let(:url) { "http://localhost:#{port}/good" }
258
- let(:event) {
259
- LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest")
260
- }
261
-
262
- subject { LogStash::Outputs::Http.new(config) }
263
-
264
- before do
265
- subject.register
266
- end
267
-
268
- describe "sending with the default (JSON) config" do
269
- let(:config) {
270
- base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1})
271
- }
272
- let(:expected_body) { LogStash::Json.dump(event) }
273
- let(:expected_content_type) { "application/json" }
274
-
275
- include_examples("a received event")
276
- end
277
-
278
- describe "sending the batch as JSON" do
279
- let(:config) do
280
- base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
281
- end
282
-
283
- let(:expected_body) { ::LogStash::Json.dump events }
284
- let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
285
- let(:expected_content_type) { "application/json" }
286
-
287
- include_examples("a received event")
288
-
289
- end
290
-
291
- describe "sending the event as a form" do
292
- let(:config) {
293
- base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})
294
- }
295
- let(:expected_body) { subject.send(:encode, event.to_hash) }
296
- let(:expected_content_type) { "application/x-www-form-urlencoded" }
297
-
298
- include_examples("a received event")
299
- end
300
-
301
- describe "sending the event as a message" do
302
- let(:config) {
303
- base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"})
304
- }
305
- let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" }
306
- let(:expected_content_type) { "text/plain" }
307
-
308
- include_examples("a received event")
309
- end
310
-
311
- describe "sending a mapped event" do
312
- let(:config) {
313
- base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} })
314
- }
315
- let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") }
316
- let(:expected_content_type) { "application/json" }
317
-
318
- include_examples("a received event")
319
- end
320
-
321
- describe "sending a mapped, nested event" do
322
- let(:config) {
323
- base_config.merge({
324
- "url" => url,
325
- "http_method" => "post",
326
- "pool_max" => 1,
327
- "mapping" => {
328
- "host" => "X %{foo}",
329
- "event" => {
330
- "user" => "Y %{user}"
331
- },
332
- "arrayevent" => [{
333
- "user" => "Z %{user}"
334
- }]
335
- }
336
- })
337
- }
338
- let(:expected_body) {
339
- LogStash::Json.dump({
340
- "host" => "X #{event.get("foo")}",
341
- "event" => {
342
- "user" => "Y #{event.get("user")}"
343
- },
344
- "arrayevent" => [{
345
- "user" => "Z #{event.get("user")}"
346
- }]
347
- })
348
- }
349
- let(:expected_content_type) { "application/json" }
350
-
351
- include_examples("a received event")
352
- end
353
- end
354
-
355
- describe "integration test without gzip compression" do
356
- include_examples("integration tests")
357
- end
358
-
359
- describe "integration test with gzip compression" do
360
- include_examples("integration tests") do
361
- let(:base_config) { { "http_compression" => true } }
362
- end
363
- end
364
- end
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/http"
3
+ require "logstash/codecs/plain"
4
+ require "thread"
5
+ require "sinatra"
6
+ require_relative "../supports/compressed_requests"
7
+
8
+ PORT = rand(65535-1024) + 1025
9
+
10
+ class LogStash::Outputs::Http
11
+ attr_writer :agent
12
+ attr_reader :request_tokens
13
+ end
14
+
15
+ # note that Sinatra startup and shutdown messages are directly logged to stderr so
16
+ # it is not really possible to disable them without reopening stderr which is not advisable.
17
+ #
18
+ # == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
19
+ # == Sinatra has ended his set (crowd applauds)
20
+ #
21
+ class TestApp < Sinatra::Base
22
+ # on the fly uncompress gzip content
23
+ use CompressedRequests
24
+
25
+ # disable WEBrick logging
26
+ def self.server_settings
27
+ { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
28
+ end
29
+
30
+ def self.multiroute(methods, path, &block)
31
+ methods.each do |method|
32
+ method.to_sym
33
+ self.send method, path, &block
34
+ end
35
+ end
36
+
37
+ def self.last_request=(request)
38
+ @last_request = request
39
+ end
40
+
41
+ def self.last_request
42
+ @last_request
43
+ end
44
+
45
+ def self.retry_fail_count=(count)
46
+ @retry_fail_count = count
47
+ end
48
+
49
+ def self.retry_fail_count()
50
+ @retry_fail_count || 2
51
+ end
52
+
53
+ multiroute(%w(get post put patch delete), "/good") do
54
+ self.class.last_request = request
55
+ [200, "YUP"]
56
+ end
57
+
58
+ multiroute(%w(get post put patch delete), "/bad") do
59
+ self.class.last_request = request
60
+ [400, "YUP"]
61
+ end
62
+
63
+ multiroute(%w(get post put patch delete), "/retry") do
64
+ self.class.last_request = request
65
+
66
+ if self.class.retry_fail_count > 0
67
+ self.class.retry_fail_count -= 1
68
+ [429, "Will succeed in #{self.class.retry_fail_count}"]
69
+ else
70
+ [200, "Done Retrying"]
71
+ end
72
+ end
73
+ end
74
+
75
+ RSpec.configure do |config|
76
+ #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
77
+ def sinatra_run_wait(app, opts)
78
+ queue = Queue.new
79
+
80
+ t = java.lang.Thread.new(
81
+ proc do
82
+ begin
83
+ app.run!(opts) do |server|
84
+ queue.push("started")
85
+ end
86
+ rescue => e
87
+ puts "Error in webserver thread #{e}"
88
+ # ignore
89
+ end
90
+ end
91
+ )
92
+ t.daemon = true
93
+ t.start
94
+ queue.pop # blocks until the run! callback runs
95
+ end
96
+
97
+ config.before(:suite) do
98
+ sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
99
+ puts "Test webserver on port #{PORT}"
100
+ end
101
+ end
102
+
103
+ describe LogStash::Outputs::Http do
104
+ # Wait for the async request to finish in this spinlock
105
+ # Requires pool_max to be 1
106
+
107
+ let(:port) { PORT }
108
+ let(:event) {
109
+ LogStash::Event.new({"message" => "hi"})
110
+ }
111
+ let(:url) { "http://localhost:#{port}/good" }
112
+ let(:method) { "post" }
113
+
114
+ shared_examples("verb behavior") do |method|
115
+ let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
116
+ subject { LogStash::Outputs::Http.new(verb_behavior_config) }
117
+
118
+ let(:expected_method) { method.clone.to_sym }
119
+ let(:client) { subject.client }
120
+
121
+ before do
122
+ subject.register
123
+ allow(client).to receive(:send).
124
+ with(expected_method, url, anything).
125
+ and_call_original
126
+ allow(subject).to receive(:log_failure).with(any_args)
127
+ allow(subject).to receive(:log_retryable_response).with(any_args)
128
+ end
129
+
130
+ context 'sending no events' do
131
+ it 'should not block the pipeline' do
132
+ subject.multi_receive([])
133
+ end
134
+ end
135
+
136
+ context "performing a get" do
137
+ describe "invoking the request" do
138
+ before do
139
+ subject.multi_receive([event])
140
+ end
141
+
142
+ it "should execute the request" do
143
+ expect(client).to have_received(:send).
144
+ with(expected_method, url, anything)
145
+ end
146
+ end
147
+
148
+ context "with passing requests" do
149
+ before do
150
+ subject.multi_receive([event])
151
+ end
152
+
153
+ it "should not log a failure" do
154
+ expect(subject).not_to have_received(:log_failure).with(any_args)
155
+ end
156
+ end
157
+
158
+ context "with failing requests" do
159
+ let(:url) { "http://localhost:#{port}/bad"}
160
+
161
+ before do
162
+ subject.multi_receive([event])
163
+ end
164
+
165
+ it "should log a failure" do
166
+ expect(subject).to have_received(:log_failure).with(any_args)
167
+ end
168
+ end
169
+
170
+ context "with ignorable failing requests" do
171
+ let(:url) { "http://localhost:#{port}/bad"}
172
+ let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
173
+
174
+ before do
175
+ subject.multi_receive([event])
176
+ end
177
+
178
+ it "should log a failure" do
179
+ expect(subject).not_to have_received(:log_failure).with(any_args)
180
+ end
181
+ end
182
+
183
+ context "with retryable failing requests" do
184
+ let(:url) { "http://localhost:#{port}/retry"}
185
+
186
+ before do
187
+ TestApp.retry_fail_count=2
188
+ allow(subject).to receive(:send_event).and_call_original
189
+ subject.multi_receive([event])
190
+ end
191
+
192
+ it "should log a retryable response 2 times" do
193
+ expect(subject).to have_received(:log_retryable_response).with(any_args).twice
194
+ end
195
+
196
+ it "should make three total requests" do
197
+ expect(subject).to have_received(:send_event).exactly(3).times
198
+ end
199
+ end
200
+
201
+ end
202
+ end
203
+
204
+ LogStash::Outputs::Http::VALID_METHODS.each do |method|
205
+ context "when using '#{method}'" do
206
+ include_examples("verb behavior", method)
207
+ end
208
+ end
209
+
210
+ shared_examples("a received event") do
211
+ before do
212
+ TestApp.last_request = nil
213
+ end
214
+
215
+ let(:events) { [event] }
216
+
217
+ describe "with a good code" do
218
+ before do
219
+ subject.multi_receive(events)
220
+ end
221
+
222
+ let(:last_request) { TestApp.last_request }
223
+ let(:body) { last_request.body.read }
224
+ let(:content_type) { last_request.env["CONTENT_TYPE"] }
225
+
226
+ it "should receive the request" do
227
+ expect(last_request).to be_truthy
228
+ end
229
+
230
+ it "should receive the event as a hash" do
231
+ expect(body).to eql(expected_body)
232
+ end
233
+
234
+ it "should have the correct content type" do
235
+ expect(content_type).to eql(expected_content_type)
236
+ end
237
+ end
238
+
239
+ describe "a retryable code" do
240
+ let(:url) { "http://localhost:#{port}/retry" }
241
+
242
+ before do
243
+ TestApp.retry_fail_count=2
244
+ allow(subject).to receive(:send_event).and_call_original
245
+ allow(subject).to receive(:log_retryable_response)
246
+ subject.multi_receive(events)
247
+ end
248
+
249
+ it "should retry" do
250
+ expect(subject).to have_received(:log_retryable_response).with(any_args).twice
251
+ end
252
+ end
253
+ end
254
+
255
+ shared_examples "integration tests" do
256
+ let(:base_config) { {} }
257
+ let(:url) { "http://localhost:#{port}/good" }
258
+ let(:event) {
259
+ LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest")
260
+ }
261
+
262
+ subject { LogStash::Outputs::Http.new(config) }
263
+
264
+ before do
265
+ subject.register
266
+ end
267
+
268
+ describe "sending with the default (JSON) config" do
269
+ let(:config) {
270
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1})
271
+ }
272
+ let(:expected_body) { LogStash::Json.dump(event) }
273
+ let(:expected_content_type) { "application/json" }
274
+
275
+ include_examples("a received event")
276
+ end
277
+
278
+ describe "sending the batch as JSON" do
279
+ let(:config) do
280
+ base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
281
+ end
282
+
283
+ let(:expected_body) { ::LogStash::Json.dump events }
284
+ let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
285
+ let(:expected_content_type) { "application/json" }
286
+
287
+ include_examples("a received event")
288
+
289
+ end
290
+
291
+ describe "sending the event as a form" do
292
+ let(:config) {
293
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})
294
+ }
295
+ let(:expected_body) { subject.send(:encode, event.to_hash) }
296
+ let(:expected_content_type) { "application/x-www-form-urlencoded" }
297
+
298
+ include_examples("a received event")
299
+ end
300
+
301
+ describe "sending the event as a message" do
302
+ let(:config) {
303
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"})
304
+ }
305
+ let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" }
306
+ let(:expected_content_type) { "text/plain" }
307
+
308
+ include_examples("a received event")
309
+ end
310
+
311
+ describe "sending a mapped event" do
312
+ let(:config) {
313
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} })
314
+ }
315
+ let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") }
316
+ let(:expected_content_type) { "application/json" }
317
+
318
+ include_examples("a received event")
319
+ end
320
+
321
+ describe "sending a mapped, nested event" do
322
+ let(:config) {
323
+ base_config.merge({
324
+ "url" => url,
325
+ "http_method" => "post",
326
+ "pool_max" => 1,
327
+ "mapping" => {
328
+ "host" => "X %{foo}",
329
+ "event" => {
330
+ "user" => "Y %{user}"
331
+ },
332
+ "arrayevent" => [{
333
+ "user" => "Z %{user}"
334
+ }]
335
+ }
336
+ })
337
+ }
338
+ let(:expected_body) {
339
+ LogStash::Json.dump({
340
+ "host" => "X #{event.get("foo")}",
341
+ "event" => {
342
+ "user" => "Y #{event.get("user")}"
343
+ },
344
+ "arrayevent" => [{
345
+ "user" => "Z #{event.get("user")}"
346
+ }]
347
+ })
348
+ }
349
+ let(:expected_content_type) { "application/json" }
350
+
351
+ include_examples("a received event")
352
+ end
353
+ end
354
+
355
+ describe "integration test without gzip compression" do
356
+ include_examples("integration tests")
357
+ end
358
+
359
+ describe "integration test with gzip compression" do
360
+ include_examples("integration tests") do
361
+ let(:base_config) { { "http_compression" => true } }
362
+ end
363
+ end
364
+ end