logstash-output-http 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2d3bfc1e5f8add9c6a24ebd8b415320ccac0e9b9
4
- data.tar.gz: 209ea0334d006933361f184ea611d9080d24c20b
3
+ metadata.gz: c0f8641077417db2ff06a2c110b1bd0db08ade0e
4
+ data.tar.gz: 1c0e81c71ec5058b354b76e8d6e895b0752c7f46
5
5
  SHA512:
6
- metadata.gz: f5015f9d8389bc52e7b0d20edd54d93b2ead727f84929c1b158cd67bf64e0e689022b68d421e97081f0971669c18008cef5f34e739238cf4ecf5000c2404bc44
7
- data.tar.gz: 2f7c7cb38cb6ef5054609424ad239d466c0faf74eea13d5533a19f8c3706fe53a15d76b022cc07f860483ca142d416520e66793ddac6b2b3f3c281c097dc6486
6
+ metadata.gz: f6fa16c1088eebbcf31f6570c889d5a99bea037cd80c1ee6e74d1b7aa3caca10d1441f6bcbc83ed6f857b1802f8220bed77964f7b47fbe7f75fa75610a36c3e7
7
+ data.tar.gz: 9565bfee79b50fb814ffbdd0ea735533070994d026033896f76b62ee5c1ccbf4fcab278a4b38cbd10dfcd94b1b3bc90137001ffb1f29d8cf3cebeef805dd66b4
data/.gitignore CHANGED
@@ -2,3 +2,5 @@
2
2
  Gemfile.lock
3
3
  .bundle
4
4
  vendor
5
+ .idea
6
+ *~
@@ -0,0 +1,4 @@
1
+ * 1.1.0
2
+ - Concurrent execution
3
+ - Add many HTTP options via the http_client mixin
4
+ - Switch to manticore as HTTP Client
@@ -2,26 +2,35 @@
2
2
  require "logstash/outputs/base"
3
3
  require "logstash/namespace"
4
4
  require "logstash/json"
5
+ require "uri"
6
+ require "logstash/plugin_mixins/http_client"
5
7
 
6
8
  class LogStash::Outputs::Http < LogStash::Outputs::Base
7
- # This output lets you `PUT` or `POST` events to a
9
+ include LogStash::PluginMixins::HttpClient
10
+
11
+ VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
12
+
13
+ # This output lets you send events to a
8
14
  # generic HTTP(S) endpoint
9
15
  #
10
- # Additionally, you are given the option to customize
11
- # the headers sent as well as basic customization of the
12
- # event json itself.
16
+ # This output will execute up to 'pool_max' requests in parallel for performance.
17
+ # Consider this when tuning this plugin for performance.
18
+ #
19
+ # Additionally, note that when parallel execution is used strict ordering of events is not
20
+ # guaranteed!
21
+ #
22
+ # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
13
23
 
14
24
  config_name "http"
15
25
 
16
26
  # URL to use
17
27
  config :url, :validate => :string, :required => :true
18
28
 
19
- # validate SSL?
20
- config :verify_ssl, :validate => :boolean, :default => true
29
+ # DEPRECATED. Set 'ssl_certificate_validation' instead
30
+ config :verify_ssl, :validate => :boolean, :default => true, :deprecated => "Please use 'ssl_certificate_validation' instead. This option will be removed in a future release!"
21
31
 
22
- # What verb to use
23
- # only put and post are supported for now
24
- config :http_method, :validate => ["put", "post"], :required => :true
32
+ # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
33
+ config :http_method, :validate => VALID_METHODS, :required => :true
25
34
 
26
35
  # Custom headers to use
27
36
  # format is `headers => ["X-My-Header", "%{host}"]`
@@ -40,7 +49,7 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
40
49
  #
41
50
  # For example:
42
51
  # [source,ruby]
43
- # mapping => ["foo", "%{host}", "bar", "%{type}"]
52
+ # mapping => {"foo", "%{host}", "bar", "%{type}"}
44
53
  config :mapping, :validate => :hash
45
54
 
46
55
  # Set the format of the http body.
@@ -55,88 +64,154 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
55
64
 
56
65
  config :message, :validate => :string
57
66
 
58
- public
59
67
  def register
60
- require "ftw"
61
- require "uri"
62
- @agent = FTW::Agent.new
63
- # TODO(sissel): SSL verify mode?
68
+ # Handle this deprecated option. TODO: remove the option
69
+ @ssl_certificate_validation = @verify_ssl if @verify_ssl
70
+ @http_method = @http_method.to_sym
71
+
72
+ # We count outstanding requests with this queue
73
+ # This queue tracks the requests to create backpressure
74
+ # When this queue is empty no new requests may be sent,
75
+ # tokens must be added back by the client on success
76
+ @request_tokens = SizedQueue.new(@pool_max)
77
+ @pool_max.times {|t| @request_tokens << true }
78
+
79
+ @requests = Array.new
64
80
 
65
81
  if @content_type.nil?
66
82
  case @format
67
83
  when "form" ; @content_type = "application/x-www-form-urlencoded"
68
84
  when "json" ; @content_type = "application/json"
85
+ when "message" ; @content_type = "text/plain"
69
86
  end
70
87
  end
71
- if @format == "message"
72
- if @message.nil?
73
- raise "message must be set if message format is used"
74
- end
75
- if @content_type.nil?
76
- raise "content_type must be set if message format is used"
77
- end
78
- unless @mapping.nil?
79
- @logger.warn "mapping is not supported and will be ignored if message format is used"
80
- end
81
- end
88
+
89
+ validate_format!
82
90
  end # def register
83
91
 
84
- public
85
92
  def receive(event)
86
93
  return unless output?(event)
87
94
 
88
- if @mapping
89
- evt = Hash.new
90
- @mapping.each do |k,v|
91
- evt[k] = event.sprintf(v)
95
+ body = event_body(event)
96
+
97
+ # Block waiting for a token
98
+ token = @request_tokens.pop
99
+
100
+ # Send the request
101
+ url = event.sprintf(@url)
102
+ headers = event_headers(event)
103
+
104
+ # Create an async request
105
+ request = client.send(@http_method, url, body: body, headers: headers, async: true)
106
+
107
+ # Invoke it using the Manticore Executor (CachedThreadPool) directly
108
+ request_async_background(request)
109
+
110
+ # Make sure we return the token to the pool
111
+ request.on_complete do
112
+ @request_tokens << token
113
+ end
114
+
115
+ request.on_success do |response|
116
+ if response.code < 200 || response.code > 299
117
+ log_failure(
118
+ "Encountered non-200 HTTP code #{200}",
119
+ :response_code => response.code,
120
+ :url => url,
121
+ :event => event)
92
122
  end
93
- else
94
- evt = event.to_hash
95
123
  end
96
124
 
97
- case @http_method
98
- when "put"
99
- request = @agent.put(event.sprintf(@url))
100
- when "post"
101
- request = @agent.post(event.sprintf(@url))
125
+ request.on_failure do |exception|
126
+ log_failure("Could not fetch URL",
127
+ :url => url,
128
+ :method => @http_method,
129
+ :body => body,
130
+ :headers => headers,
131
+ :message => exception.message,
132
+ :class => exception.class.name,
133
+ :backtrace => exception.backtrace
134
+ )
135
+ end
136
+ end
137
+
138
+ private
139
+
140
+ # This is split into a separate method mostly to help testing
141
+ def log_failure(message, opts)
142
+ @logger.error("[HTTP Output Failure] #{message}", opts)
143
+ end
144
+
145
+ # Manticore doesn't provide a way to attach handlers to background or async requests well
146
+ # It wants you to use futures. The #async method kinda works but expects single thread batches
147
+ # and background only returns futures.
148
+ # Proposed fix to manticore here: https://github.com/cheald/manticore/issues/32
149
+ def request_async_background(request)
150
+ @method ||= client.executor.java_method(:submit, [java.util.concurrent.Callable.java_class])
151
+ @method.call(request)
152
+ end
153
+
154
+ # Format the HTTP body
155
+ def event_body(event)
156
+ # TODO: Create an HTTP post data codec, use that here
157
+ if @format == "json"
158
+ LogStash::Json.dump(map_event(event))
159
+ elsif @format == "message"
160
+ event.sprintf(@message)
102
161
  else
103
- @logger.error("Unknown verb:", :verb => @http_method)
162
+ encode(map_event(event))
104
163
  end
164
+ end
105
165
 
106
- if @headers
107
- @headers.each do |k,v|
108
- request.headers[k] = event.sprintf(v)
166
+ def map_event(event)
167
+ if @mapping
168
+ @mapping.reduce({}) do |acc,kv|
169
+ k,v = kv
170
+ acc[k] = event.sprintf(v)
171
+ acc
109
172
  end
173
+ else
174
+ event.to_hash
110
175
  end
176
+ end
111
177
 
112
- request["Content-Type"] = @content_type
178
+ def event_headers(event)
179
+ headers = custom_headers(event) || {}
180
+ headers["Content-Type"] = @content_type
181
+ headers
182
+ end
113
183
 
114
- begin
115
- if @format == "json"
116
- request.body = LogStash::Json.dump(evt)
117
- elsif @format == "message"
118
- request.body = event.sprintf(@message)
119
- else
120
- request.body = encode(evt)
121
- end
122
- #puts "#{request.port} / #{request.protocol}"
123
- #puts request
124
- #puts
125
- #puts request.body
126
- response = @agent.execute(request)
127
-
128
- # Consume body to let this connection be reused
129
- rbody = ""
130
- response.read_body { |c| rbody << c }
131
- #puts rbody
132
- rescue Exception => e
133
- @logger.warn("Unhandled exception", :request => request, :response => response, :exception => e, :stacktrace => e.backtrace)
184
+ def custom_headers(event)
185
+ return nil unless @headers
186
+
187
+ @headers.reduce({}) do |acc,kv|
188
+ k,v = kv
189
+ acc[k] = event.sprintf(v)
190
+ acc
134
191
  end
135
- end # def receive
192
+ end
136
193
 
194
+ #TODO Extract this to a codec
137
195
  def encode(hash)
138
196
  return hash.collect do |key, value|
139
- CGI.escape(key) + "=" + CGI.escape(value)
197
+ CGI.escape(key) + "=" + CGI.escape(value.to_s)
140
198
  end.join("&")
141
- end # def encode
199
+ end
200
+
201
+
202
+ def validate_format!
203
+ if @format == "message"
204
+ if @message.nil?
205
+ raise "message must be set if message format is used"
206
+ end
207
+
208
+ if @content_type.nil?
209
+ raise "content_type must be set if message format is used"
210
+ end
211
+
212
+ unless @mapping.nil?
213
+ @logger.warn "mapping is not supported and will be ignored if message format is used"
214
+ end
215
+ end
216
+ end
142
217
  end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-http'
4
- s.version = '1.0.0'
4
+ s.version = '1.1.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint"
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,9 +21,11 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
24
-
25
- s.add_runtime_dependency 'ftw', ['~> 0.0.40']
24
+ s.add_runtime_dependency 'logstash-mixin-http_client', '>= 1.0.1', '< 2.0.0'
26
25
 
27
26
  s.add_development_dependency 'logstash-devutils'
27
+ s.add_development_dependency 'logstash-codec-plain'
28
+ s.add_development_dependency 'sinatra'
29
+ s.add_development_dependency 'webrick'
28
30
  end
29
31
 
@@ -1 +1,232 @@
1
1
  require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/outputs/http"
3
+ require "thread"
4
+ require "sinatra"
5
+
6
+ PORT = rand(65535-1024) + 1025
7
+
8
+ class LogStash::Outputs::Http
9
+ attr_writer :agent
10
+ attr_reader :request_tokens
11
+ end
12
+
13
+ class TestApp < Sinatra::Base
14
+ def self.multiroute(methods, path, &block)
15
+ methods.each do |method|
16
+ method.to_sym
17
+ self.send method, path, &block
18
+ end
19
+ end
20
+
21
+ def self.last_request=(request)
22
+ @last_request = request
23
+ end
24
+
25
+ def self.last_request
26
+ @last_request
27
+ end
28
+
29
+ multiroute(%w(get post put patch delete), "/good") do
30
+ self.class.last_request = request
31
+ [200, "YUP"]
32
+ end
33
+
34
+ multiroute(%w(get post put patch delete), "/bad") do
35
+ self.class.last_request = request
36
+ [500, "YUP"]
37
+ end
38
+ end
39
+
40
+ RSpec.configure do |config|
41
+ #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
42
+ def sinatra_run_wait(app, opts)
43
+ queue = Queue.new
44
+ thread = Thread.new do
45
+ Thread.abort_on_exception = true
46
+ app.run!(opts) do |server|
47
+ queue.push("started")
48
+ end
49
+ end
50
+ queue.pop # blocks until the run! callback runs
51
+ end
52
+
53
+
54
+ config.before(:suite) do
55
+ sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
56
+ end
57
+ end
58
+
59
+ describe LogStash::Outputs::Http do
60
+ # Wait for the async request to finish in this spinlock
61
+ # Requires pool_max to be 1
62
+ def wait_for_request
63
+
64
+ loop do
65
+ break if subject.request_tokens.size > 0
66
+ end
67
+ end
68
+
69
+ let(:port) { PORT }
70
+ let(:event) { LogStash::Event.new("message" => "hi") }
71
+ let(:url) { "http://localhost:#{port}/good" }
72
+ let(:method) { "post" }
73
+
74
+ describe "when num requests > token count" do
75
+ let(:pool_max) { 10 }
76
+ let(:num_reqs) { pool_max / 2 }
77
+ let(:client) { subject.client }
78
+ subject {
79
+ LogStash::Outputs::Http.new("url" => url,
80
+ "http_method" => method,
81
+ "pool_max" => pool_max)
82
+ }
83
+
84
+ before do
85
+ subject.register
86
+ end
87
+
88
+ it "should receive all the requests" do
89
+ expect(client).to receive(:send).
90
+ with(method.to_sym, url, anything).
91
+ exactly(num_reqs).times.
92
+ and_call_original
93
+
94
+ num_reqs.times {|t| subject.receive(event)}
95
+ end
96
+ end
97
+
98
+ shared_examples("verb behavior") do |method|
99
+ subject { LogStash::Outputs::Http.new("url" => url, "http_method" => method, "pool_max" => 1) }
100
+
101
+ let(:expected_method) { method.clone.to_sym }
102
+ let(:client) { subject.client }
103
+
104
+ before do
105
+ subject.register
106
+ allow(client).to receive(:send).
107
+ with(expected_method, url, anything).
108
+ and_call_original
109
+ allow(subject).to receive(:log_failure).with(any_args)
110
+ end
111
+
112
+ context "performing a get" do
113
+ describe "invoking the request" do
114
+ before do
115
+ subject.receive(event)
116
+ end
117
+
118
+ it "should execute the request" do
119
+ expect(client).to have_received(:send).
120
+ with(expected_method, url, anything)
121
+ end
122
+ end
123
+
124
+ context "with passing requests" do
125
+ before do
126
+ subject.receive(event)
127
+ end
128
+
129
+ it "should not log a failure" do
130
+ expect(subject).not_to have_received(:log_failure).with(any_args)
131
+ end
132
+ end
133
+
134
+ context "with failing requests" do
135
+ let(:url) { "http://localhost:#{port}/bad"}
136
+
137
+ before do
138
+ subject.receive(event)
139
+ wait_for_request
140
+ end
141
+
142
+ it "should log a failure" do
143
+ expect(subject).to have_received(:log_failure).with(any_args)
144
+ end
145
+ end
146
+ end
147
+ end
148
+
149
+ LogStash::Outputs::Http::VALID_METHODS.each do |method|
150
+ context "when using '#{method}'" do
151
+ include_examples("verb behavior", method)
152
+ end
153
+ end
154
+
155
+ shared_examples("a received event") do
156
+ before do
157
+ TestApp.last_request = nil
158
+ end
159
+
160
+ before do
161
+ subject.receive(event)
162
+ wait_for_request
163
+ end
164
+
165
+ let(:last_request) { TestApp.last_request }
166
+ let(:body) { last_request.body.read }
167
+ let(:content_type) { last_request.env["CONTENT_TYPE"] }
168
+
169
+ it "should receive the request" do
170
+ expect(last_request).to be_truthy
171
+ end
172
+
173
+ it "should receive the event as a hash" do
174
+ expect(body).to eql(expected_body)
175
+ end
176
+
177
+ it "should have the correct content type" do
178
+ expect(content_type).to eql(expected_content_type)
179
+ end
180
+ end
181
+
182
+ describe "integration tests" do
183
+ let(:url) { "http://localhost:#{port}/good" }
184
+ let(:event) { LogStash::Event.new("foo" => "bar", "baz" => "bot")}
185
+
186
+ subject { LogStash::Outputs::Http.new(config) }
187
+
188
+ before do
189
+ subject.register
190
+ end
191
+
192
+ describe "sending with the default (JSON) config" do
193
+ let(:config) {
194
+ {"url" => url, "http_method" => "post", "pool_max" => 1}
195
+ }
196
+ let(:expected_body) { LogStash::Json.dump(event) }
197
+ let(:expected_content_type) { "application/json" }
198
+
199
+ include_examples("a received event")
200
+ end
201
+
202
+ describe "sending the event as a form" do
203
+ let(:config) {
204
+ {"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"}
205
+ }
206
+ let(:expected_body) { subject.send(:encode, event.to_hash) }
207
+ let(:expected_content_type) { "application/x-www-form-urlencoded" }
208
+
209
+ include_examples("a received event")
210
+ end
211
+
212
+ describe "sending the event as a message" do
213
+ let(:config) {
214
+ {"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"}
215
+ }
216
+ let(:expected_body) { "#{event["foo"]} AND #{event["baz"]}" }
217
+ let(:expected_content_type) { "text/plain" }
218
+
219
+ include_examples("a received event")
220
+ end
221
+
222
+ describe "sending a mapped event" do
223
+ let(:config) {
224
+ {"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"}}
225
+ }
226
+ let(:expected_body) { LogStash::Json.dump("blah" => "X #{event["foo"]}") }
227
+ let(:expected_content_type) { "application/json" }
228
+
229
+ include_examples("a received event")
230
+ end
231
+ end
232
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-24 00:00:00.000000000 Z
11
+ date: 2015-08-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
@@ -31,17 +31,23 @@ dependencies:
31
31
  prerelease: false
32
32
  type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
- name: ftw
34
+ name: logstash-mixin-http_client
35
35
  version_requirements: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ~>
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: 1.0.1
40
+ - - <
38
41
  - !ruby/object:Gem::Version
39
- version: 0.0.40
42
+ version: 2.0.0
40
43
  requirement: !ruby/object:Gem::Requirement
41
44
  requirements:
42
- - - ~>
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: 1.0.1
48
+ - - <
43
49
  - !ruby/object:Gem::Version
44
- version: 0.0.40
50
+ version: 2.0.0
45
51
  prerelease: false
46
52
  type: :runtime
47
53
  - !ruby/object:Gem::Dependency
@@ -58,6 +64,48 @@ dependencies:
58
64
  version: '0'
59
65
  prerelease: false
60
66
  type: :development
67
+ - !ruby/object:Gem::Dependency
68
+ name: logstash-codec-plain
69
+ version_requirements: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - '>='
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ requirement: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - '>='
77
+ - !ruby/object:Gem::Version
78
+ version: '0'
79
+ prerelease: false
80
+ type: :development
81
+ - !ruby/object:Gem::Dependency
82
+ name: sinatra
83
+ version_requirements: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - '>='
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ requirement: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - '>='
91
+ - !ruby/object:Gem::Version
92
+ version: '0'
93
+ prerelease: false
94
+ type: :development
95
+ - !ruby/object:Gem::Dependency
96
+ name: webrick
97
+ version_requirements: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ requirement: !ruby/object:Gem::Requirement
103
+ requirements:
104
+ - - '>='
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ prerelease: false
108
+ type: :development
61
109
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
62
110
  email: info@elastic.co
63
111
  executables: []
@@ -97,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
97
145
  version: '0'
98
146
  requirements: []
99
147
  rubyforge_project:
100
- rubygems_version: 2.2.2
148
+ rubygems_version: 2.1.9
101
149
  signing_key:
102
150
  specification_version: 4
103
151
  summary: This output lets you `PUT` or `POST` events to a generic HTTP(S) endpoint