logstash-output-httpcodec 5.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,28 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-httpcodec'
3
+ s.version = '5.2.5'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = "Sends events to a generic HTTP or HTTPS endpoint"
6
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Elastic"]
8
+ s.email = 'info@elastic.co'
9
+ s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
14
+
15
+ # Tests
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+
18
+ # Special flag to let us know this is actually a logstash plugin
19
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
20
+
21
+ # Gem dependencies
22
+ s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
23
+ s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
24
+
25
+ s.add_development_dependency 'logstash-devutils'
26
+ s.add_development_dependency 'sinatra'
27
+ s.add_development_dependency 'webrick'
28
+ end
@@ -0,0 +1,364 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/httpcodec"
3
+ require "logstash/codecs/plain"
4
+ require "thread"
5
+ require "sinatra"
6
+ require_relative "../supports/compressed_requests"
7
+
8
+ PORT = rand(65535-1024) + 1025
9
+
10
+ class LogStash::Outputs::HttpCodec
11
+ attr_writer :agent
12
+ attr_reader :request_tokens
13
+ end
14
+
15
+ # note that Sinatra startup and shutdown messages are directly logged to stderr so
16
+ # it is not really possible to disable them without reopening stderr which is not advisable.
17
+ #
18
+ # == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
19
+ # == Sinatra has ended his set (crowd applauds)
20
+ #
21
+ class TestApp < Sinatra::Base
22
+ # on the fly uncompress gzip content
23
+ use CompressedRequests
24
+
25
+ # disable WEBrick logging
26
+ def self.server_settings
27
+ { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
28
+ end
29
+
30
+ def self.multiroute(methods, path, &block)
31
+ methods.each do |method|
32
+ method.to_sym
33
+ self.send method, path, &block
34
+ end
35
+ end
36
+
37
+ def self.last_request=(request)
38
+ @last_request = request
39
+ end
40
+
41
+ def self.last_request
42
+ @last_request
43
+ end
44
+
45
+ def self.retry_fail_count=(count)
46
+ @retry_fail_count = count
47
+ end
48
+
49
+ def self.retry_fail_count()
50
+ @retry_fail_count || 2
51
+ end
52
+
53
+ multiroute(%w(get post put patch delete), "/good") do
54
+ self.class.last_request = request
55
+ [200, "YUP"]
56
+ end
57
+
58
+ multiroute(%w(get post put patch delete), "/bad") do
59
+ self.class.last_request = request
60
+ [400, "YUP"]
61
+ end
62
+
63
+ multiroute(%w(get post put patch delete), "/retry") do
64
+ self.class.last_request = request
65
+
66
+ if self.class.retry_fail_count > 0
67
+ self.class.retry_fail_count -= 1
68
+ [429, "Will succeed in #{self.class.retry_fail_count}"]
69
+ else
70
+ [200, "Done Retrying"]
71
+ end
72
+ end
73
+ end
74
+
75
+ RSpec.configure do |config|
76
+ #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
77
+ def sinatra_run_wait(app, opts)
78
+ queue = Queue.new
79
+
80
+ t = java.lang.Thread.new(
81
+ proc do
82
+ begin
83
+ app.run!(opts) do |server|
84
+ queue.push("started")
85
+ end
86
+ rescue => e
87
+ puts "Error in webserver thread #{e}"
88
+ # ignore
89
+ end
90
+ end
91
+ )
92
+ t.daemon = true
93
+ t.start
94
+ queue.pop # blocks until the run! callback runs
95
+ end
96
+
97
+ config.before(:suite) do
98
+ sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
99
+ puts "Test webserver on port #{PORT}"
100
+ end
101
+ end
102
+
103
+ describe LogStash::Outputs::HttpCodec do
104
+ # Wait for the async request to finish in this spinlock
105
+ # Requires pool_max to be 1
106
+
107
+ let(:port) { PORT }
108
+ let(:event) {
109
+ LogStash::Event.new({"message" => "hi"})
110
+ }
111
+ let(:url) { "http://localhost:#{port}/good" }
112
+ let(:method) { "post" }
113
+
114
+ shared_examples("verb behavior") do |method|
115
+ let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
116
+ subject { LogStash::Outputs::HttpCodec.new(verb_behavior_config) }
117
+
118
+ let(:expected_method) { method.clone.to_sym }
119
+ let(:client) { subject.client }
120
+
121
+ before do
122
+ subject.register
123
+ allow(client).to receive(:send).
124
+ with(expected_method, url, anything).
125
+ and_call_original
126
+ allow(subject).to receive(:log_failure).with(any_args)
127
+ allow(subject).to receive(:log_retryable_response).with(any_args)
128
+ end
129
+
130
+ context 'sending no events' do
131
+ it 'should not block the pipeline' do
132
+ subject.multi_receive([])
133
+ end
134
+ end
135
+
136
+ context "performing a get" do
137
+ describe "invoking the request" do
138
+ before do
139
+ subject.multi_receive([event])
140
+ end
141
+
142
+ it "should execute the request" do
143
+ expect(client).to have_received(:send).
144
+ with(expected_method, url, anything)
145
+ end
146
+ end
147
+
148
+ context "with passing requests" do
149
+ before do
150
+ subject.multi_receive([event])
151
+ end
152
+
153
+ it "should not log a failure" do
154
+ expect(subject).not_to have_received(:log_failure).with(any_args)
155
+ end
156
+ end
157
+
158
+ context "with failing requests" do
159
+ let(:url) { "http://localhost:#{port}/bad"}
160
+
161
+ before do
162
+ subject.multi_receive([event])
163
+ end
164
+
165
+ it "should log a failure" do
166
+ expect(subject).to have_received(:log_failure).with(any_args)
167
+ end
168
+ end
169
+
170
+ context "with ignorable failing requests" do
171
+ let(:url) { "http://localhost:#{port}/bad"}
172
+ let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) }
173
+
174
+ before do
175
+ subject.multi_receive([event])
176
+ end
177
+
178
+ it "should log a failure" do
179
+ expect(subject).not_to have_received(:log_failure).with(any_args)
180
+ end
181
+ end
182
+
183
+ context "with retryable failing requests" do
184
+ let(:url) { "http://localhost:#{port}/retry"}
185
+
186
+ before do
187
+ TestApp.retry_fail_count=2
188
+ allow(subject).to receive(:send_event).and_call_original
189
+ subject.multi_receive([event])
190
+ end
191
+
192
+ it "should log a retryable response 2 times" do
193
+ expect(subject).to have_received(:log_retryable_response).with(any_args).twice
194
+ end
195
+
196
+ it "should make three total requests" do
197
+ expect(subject).to have_received(:send_event).exactly(3).times
198
+ end
199
+ end
200
+
201
+ end
202
+ end
203
+
204
+ LogStash::Outputs::HttpCodec::VALID_METHODS.each do |method|
205
+ context "when using '#{method}'" do
206
+ include_examples("verb behavior", method)
207
+ end
208
+ end
209
+
210
+ shared_examples("a received event") do
211
+ before do
212
+ TestApp.last_request = nil
213
+ end
214
+
215
+ let(:events) { [event] }
216
+
217
+ describe "with a good code" do
218
+ before do
219
+ subject.multi_receive(events)
220
+ end
221
+
222
+ let(:last_request) { TestApp.last_request }
223
+ let(:body) { last_request.body.read }
224
+ let(:content_type) { last_request.env["CONTENT_TYPE"] }
225
+
226
+ it "should receive the request" do
227
+ expect(last_request).to be_truthy
228
+ end
229
+
230
+ it "should receive the event as a hash" do
231
+ expect(body).to eql(expected_body)
232
+ end
233
+
234
+ it "should have the correct content type" do
235
+ expect(content_type).to eql(expected_content_type)
236
+ end
237
+ end
238
+
239
+ describe "a retryable code" do
240
+ let(:url) { "http://localhost:#{port}/retry" }
241
+
242
+ before do
243
+ TestApp.retry_fail_count=2
244
+ allow(subject).to receive(:send_event).and_call_original
245
+ allow(subject).to receive(:log_retryable_response)
246
+ subject.multi_receive(events)
247
+ end
248
+
249
+ it "should retry" do
250
+ expect(subject).to have_received(:log_retryable_response).with(any_args).twice
251
+ end
252
+ end
253
+ end
254
+
255
+ shared_examples "integration tests" do
256
+ let(:base_config) { {} }
257
+ let(:url) { "http://localhost:#{port}/good" }
258
+ let(:event) {
259
+ LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest")
260
+ }
261
+
262
+ subject { LogStash::Outputs::HttpCodec.new(config) }
263
+
264
+ before do
265
+ subject.register
266
+ end
267
+
268
+ describe "sending with the default (JSON) config" do
269
+ let(:config) {
270
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1})
271
+ }
272
+ let(:expected_body) { LogStash::Json.dump(event) }
273
+ let(:expected_content_type) { "application/json" }
274
+
275
+ include_examples("a received event")
276
+ end
277
+
278
+ describe "sending the batch as JSON" do
279
+ let(:config) do
280
+ base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"})
281
+ end
282
+
283
+ let(:expected_body) { ::LogStash::Json.dump events }
284
+ let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]}
285
+ let(:expected_content_type) { "application/json" }
286
+
287
+ include_examples("a received event")
288
+
289
+ end
290
+
291
+ describe "sending the event as a form" do
292
+ let(:config) {
293
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"})
294
+ }
295
+ let(:expected_body) { subject.send(:encode, event.to_hash) }
296
+ let(:expected_content_type) { "application/x-www-form-urlencoded" }
297
+
298
+ include_examples("a received event")
299
+ end
300
+
301
+ describe "sending the event as a message" do
302
+ let(:config) {
303
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"})
304
+ }
305
+ let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" }
306
+ let(:expected_content_type) { "text/plain" }
307
+
308
+ include_examples("a received event")
309
+ end
310
+
311
+ describe "sending a mapped event" do
312
+ let(:config) {
313
+ base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} })
314
+ }
315
+ let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") }
316
+ let(:expected_content_type) { "application/json" }
317
+
318
+ include_examples("a received event")
319
+ end
320
+
321
+ describe "sending a mapped, nested event" do
322
+ let(:config) {
323
+ base_config.merge({
324
+ "url" => url,
325
+ "http_method" => "post",
326
+ "pool_max" => 1,
327
+ "mapping" => {
328
+ "host" => "X %{foo}",
329
+ "event" => {
330
+ "user" => "Y %{user}"
331
+ },
332
+ "arrayevent" => [{
333
+ "user" => "Z %{user}"
334
+ }]
335
+ }
336
+ })
337
+ }
338
+ let(:expected_body) {
339
+ LogStash::Json.dump({
340
+ "host" => "X #{event.get("foo")}",
341
+ "event" => {
342
+ "user" => "Y #{event.get("user")}"
343
+ },
344
+ "arrayevent" => [{
345
+ "user" => "Z #{event.get("user")}"
346
+ }]
347
+ })
348
+ }
349
+ let(:expected_content_type) { "application/json" }
350
+
351
+ include_examples("a received event")
352
+ end
353
+ end
354
+
355
+ describe "integration test without gzip compression" do
356
+ include_examples("integration tests")
357
+ end
358
+
359
+ describe "integration test with gzip compression" do
360
+ include_examples("integration tests") do
361
+ let(:base_config) { { "http_compression" => true } }
362
+ end
363
+ end
364
+ end
@@ -0,0 +1,38 @@
1
+ # encoding: utf-8
2
+ #
3
+ # based on relistan's rack handler
4
+ # out of the box rack only gives use the rack deflater handler to return compressed
5
+ # response, this gist offer the inverse and should work on all rack based app like sinatra or rails.
6
+ #
7
+ # original source: https://gist.github.com/relistan/2109707
8
+ require "zlib"
9
+
10
+ class CompressedRequests
11
+ def initialize(app)
12
+ @app = app
13
+ end
14
+
15
+ def encoding_handled?(env)
16
+ ['gzip', 'deflate'].include? env['HTTP_CONTENT_ENCODING']
17
+ end
18
+
19
+ def call(env)
20
+ if encoding_handled?(env)
21
+ extracted = decode(env['rack.input'], env['HTTP_CONTENT_ENCODING'])
22
+
23
+ env.delete('HTTP_CONTENT_ENCODING')
24
+ env['CONTENT_LENGTH'] = extracted.bytesize
25
+ env['rack.input'] = StringIO.new(extracted)
26
+ end
27
+
28
+ status, headers, response = @app.call(env)
29
+ return [status, headers, response]
30
+ end
31
+
32
+ def decode(input, content_encoding)
33
+ case content_encoding
34
+ when 'gzip' then Zlib::GzipReader.new(input).read
35
+ when 'deflate' then Zlib::Inflate.inflate(input.read)
36
+ end
37
+ end
38
+ end
metadata ADDED
@@ -0,0 +1,142 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-output-httpcodec
3
+ version: !ruby/object:Gem::Version
4
+ version: 5.2.5
5
+ platform: ruby
6
+ authors:
7
+ - Elastic
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-05-24 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '1.60'
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
23
+ prerelease: false
24
+ type: :runtime
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.60'
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: 6.0.0
39
+ - - "<"
40
+ - !ruby/object:Gem::Version
41
+ version: 8.0.0
42
+ name: logstash-mixin-http_client
43
+ prerelease: false
44
+ type: :runtime
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ version: 6.0.0
50
+ - - "<"
51
+ - !ruby/object:Gem::Version
52
+ version: 8.0.0
53
+ - !ruby/object:Gem::Dependency
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - ">="
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ name: logstash-devutils
60
+ prerelease: false
61
+ type: :development
62
+ version_requirements: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ - !ruby/object:Gem::Dependency
68
+ requirement: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ name: sinatra
74
+ prerelease: false
75
+ type: :development
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ - !ruby/object:Gem::Dependency
82
+ requirement: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ name: webrick
88
+ prerelease: false
89
+ type: :development
90
+ version_requirements: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ description: This gem is a Logstash plugin required to be installed on top of the
96
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
97
+ gem is not a stand-alone program
98
+ email: info@elastic.co
99
+ executables: []
100
+ extensions: []
101
+ extra_rdoc_files: []
102
+ files:
103
+ - CHANGELOG.md
104
+ - CONTRIBUTORS
105
+ - Gemfile
106
+ - LICENSE
107
+ - NOTICE.TXT
108
+ - README.md
109
+ - docs/index.asciidoc
110
+ - lib/logstash/outputs/httpcodec.rb
111
+ - logstash-output-httpcodec.gemspec
112
+ - spec/outputs/httpcodec_spec.rb
113
+ - spec/supports/compressed_requests.rb
114
+ homepage: http://www.elastic.co/guide/en/logstash/current/index.html
115
+ licenses:
116
+ - Apache License (2.0)
117
+ metadata:
118
+ logstash_plugin: 'true'
119
+ logstash_group: output
120
+ post_install_message:
121
+ rdoc_options: []
122
+ require_paths:
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ version: '0'
129
+ required_rubygems_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ requirements: []
135
+ rubyforge_project:
136
+ rubygems_version: 2.7.10
137
+ signing_key:
138
+ specification_version: 4
139
+ summary: Sends events to a generic HTTP or HTTPS endpoint
140
+ test_files:
141
+ - spec/outputs/httpcodec_spec.rb
142
+ - spec/supports/compressed_requests.rb