fluent-plugin-out-http 0.1.4 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +4 -4
- data/CHANGELOG.md +12 -0
- data/fluent-plugin-out-http.gemspec +3 -1
- data/lib/fluent/plugin/out_http.rb +25 -6
- data/lib/fluent/test/http_output_test.rb +1 -13
- data/test/plugin/test_out_http.rb +105 -35
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 12f3735f1d540b81b847e6fadf5bdca62cf963bf
|
4
|
+
data.tar.gz: 75df66f76592807c49682def91dd3a88374e13f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9bee4d8fac451d23d10f84fc2e1458b848d44fb70a8a3521b5a8b23f18c8101b480316c1bafcf5783091fc5e659ed915b74cd0da6cdc30e693bfdfae72d7f3dc
|
7
|
+
data.tar.gz: eb73a64c59eb8c690bc9d5750c2f22e3e67356a26ebf30bdf67593d4df99f2fda2f4dc355b67624a5c1f0fbd737bcfa6ede122962da00277c4baaf90990b003e
|
data/.travis.yml
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
rvm:
|
2
|
-
- 1.9.3
|
3
|
-
- 2.0.0
|
4
2
|
- 2.1
|
5
3
|
- 2.2
|
4
|
+
- 2.3
|
5
|
+
- 2.4.0
|
6
6
|
- ruby-head
|
7
|
-
- rbx-2
|
8
7
|
|
9
8
|
os:
|
10
9
|
- linux
|
11
10
|
- osx
|
12
11
|
|
12
|
+
dist: trusty
|
13
|
+
|
13
14
|
gemfile:
|
14
15
|
- Gemfile
|
15
16
|
|
@@ -18,4 +19,3 @@ script: bundle exec rake test
|
|
18
19
|
matrix:
|
19
20
|
allow_failures:
|
20
21
|
- rvm: ruby-head
|
21
|
-
- rvm: rbx-2
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 0.2.0
|
4
|
+
### Added
|
5
|
+
* SSL is now supported if `endpoint_url` uses the `https` scheme (uses ruby-2.1 syntax internally)
|
6
|
+
* New config: set `ssl_no_verify` to `true` to bypass SSL certificate verification.
|
7
|
+
Use at your own risk.
|
8
|
+
### Changed
|
9
|
+
* Fixed tests:
|
10
|
+
* Removed some warnings
|
11
|
+
* Fixed failing binary test to use UTF-8
|
12
|
+
### Removed
|
13
|
+
* Dropped support of Ruby 1.9-2.0
|
14
|
+
|
3
15
|
## 0.1.4
|
4
16
|
* #11 Updated Fluentd dependency to: [">= 0.10.0", "< 2"]
|
5
17
|
* #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-out-http"
|
5
|
-
gem.version = "0.
|
5
|
+
gem.version = "0.2.0"
|
6
6
|
gem.authors = ["Marica Odagaki"]
|
7
7
|
gem.email = ["ento.entotto@gmail.com"]
|
8
8
|
gem.summary = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint}
|
@@ -15,6 +15,8 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
|
18
|
+
gem.required_ruby_version = '>= 2.1.0'
|
19
|
+
|
18
20
|
gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
|
19
21
|
gem.add_runtime_dependency "fluentd", [">= 0.10.0", "< 2"]
|
20
22
|
gem.add_development_dependency "bundler"
|
@@ -8,12 +8,15 @@ class Fluent::HTTPOutput < Fluent::Output
|
|
8
8
|
require 'yajl'
|
9
9
|
end
|
10
10
|
|
11
|
-
# Endpoint URL ex. localhost.local/api/
|
11
|
+
# Endpoint URL ex. http://localhost.local/api/
|
12
12
|
config_param :endpoint_url, :string
|
13
13
|
|
14
|
+
# Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
|
15
|
+
config_param :ssl_no_verify, :bool, :default => false
|
16
|
+
|
14
17
|
# HTTP method
|
15
18
|
config_param :http_method, :string, :default => :post
|
16
|
-
|
19
|
+
|
17
20
|
# form | json
|
18
21
|
config_param :serializer, :string, :default => :form
|
19
22
|
|
@@ -25,13 +28,19 @@ class Fluent::HTTPOutput < Fluent::Output
|
|
25
28
|
config_param :raise_on_error, :bool, :default => true
|
26
29
|
|
27
30
|
# nil | 'none' | 'basic'
|
28
|
-
config_param :authentication, :string, :default => nil
|
31
|
+
config_param :authentication, :string, :default => nil
|
29
32
|
config_param :username, :string, :default => ''
|
30
33
|
config_param :password, :string, :default => '', :secret => true
|
31
34
|
|
32
35
|
def configure(conf)
|
33
36
|
super
|
34
37
|
|
38
|
+
@ssl_verify_mode = if @ssl_no_verify
|
39
|
+
OpenSSL::SSL::VERIFY_NONE
|
40
|
+
else
|
41
|
+
OpenSSL::SSL::VERIFY_PEER
|
42
|
+
end
|
43
|
+
|
35
44
|
serializers = [:json, :form]
|
36
45
|
@serializer = if serializers.include? @serializer.intern
|
37
46
|
@serializer.intern
|
@@ -51,6 +60,8 @@ class Fluent::HTTPOutput < Fluent::Output
|
|
51
60
|
else
|
52
61
|
:none
|
53
62
|
end
|
63
|
+
|
64
|
+
@last_request_time = nil
|
54
65
|
end
|
55
66
|
|
56
67
|
def start
|
@@ -92,13 +103,21 @@ class Fluent::HTTPOutput < Fluent::Output
|
|
92
103
|
return req, uri
|
93
104
|
end
|
94
105
|
|
95
|
-
def
|
106
|
+
def http_opts(uri)
|
107
|
+
opts = {
|
108
|
+
:use_ssl => uri.scheme == 'https'
|
109
|
+
}
|
110
|
+
opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
|
111
|
+
opts
|
112
|
+
end
|
113
|
+
|
114
|
+
def send_request(req, uri)
|
96
115
|
is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
|
97
116
|
if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
|
98
117
|
$log.info('Dropped request due to rate limiting')
|
99
118
|
return
|
100
119
|
end
|
101
|
-
|
120
|
+
|
102
121
|
res = nil
|
103
122
|
|
104
123
|
begin
|
@@ -106,7 +125,7 @@ class Fluent::HTTPOutput < Fluent::Output
|
|
106
125
|
req.basic_auth(@username, @password)
|
107
126
|
end
|
108
127
|
@last_request_time = Time.now.to_f
|
109
|
-
res = Net::HTTP.
|
128
|
+
res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
|
110
129
|
rescue => e # rescue all StandardErrors
|
111
130
|
# server didn't respond
|
112
131
|
$log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
|
@@ -30,6 +30,7 @@ class Test::Unit::TestCase
|
|
30
30
|
end
|
31
31
|
|
32
32
|
require 'webrick'
|
33
|
+
require 'webrick/https'
|
33
34
|
|
34
35
|
# to handle POST/PUT/DELETE ...
|
35
36
|
module WEBrick::HTTPServlet
|
@@ -39,16 +40,3 @@ module WEBrick::HTTPServlet
|
|
39
40
|
alias do_DELETE do_GET
|
40
41
|
end
|
41
42
|
end
|
42
|
-
|
43
|
-
def get_code(server, port, path, headers={})
|
44
|
-
require 'net/http'
|
45
|
-
Net::HTTP.start(server, port){|http|
|
46
|
-
http.get(path, headers).code
|
47
|
-
}
|
48
|
-
end
|
49
|
-
def get_content(server, port, path, headers={})
|
50
|
-
require 'net/http'
|
51
|
-
Net::HTTP.start(server, port){|http|
|
52
|
-
http.get(path, headers).body
|
53
|
-
}
|
54
|
-
end
|
@@ -1,14 +1,31 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
+
require 'net/http'
|
2
3
|
require 'uri'
|
3
4
|
require 'yajl'
|
4
5
|
require 'fluent/test/http_output_test'
|
5
6
|
require 'fluent/plugin/out_http'
|
6
7
|
|
7
8
|
|
8
|
-
|
9
|
+
class HTTPOutputTestBase < Test::Unit::TestCase
|
10
|
+
def self.port
|
11
|
+
5126
|
12
|
+
end
|
9
13
|
|
14
|
+
def self.server_config
|
15
|
+
config = {BindAddress: '127.0.0.1', Port: port}
|
16
|
+
if ENV['VERBOSE']
|
17
|
+
logger = WEBrick::Log.new(STDOUT, WEBrick::BasicLog::DEBUG)
|
18
|
+
config[:Logger] = logger
|
19
|
+
config[:AccessLog] = []
|
20
|
+
end
|
21
|
+
config
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.test_http_client(**opts)
|
25
|
+
opts = opts.merge(open_timeout: 1, read_timeout: 1)
|
26
|
+
Net::HTTP.start('127.0.0.1', port, **opts)
|
27
|
+
end
|
10
28
|
|
11
|
-
class HTTPOutputTestBase < Test::Unit::TestCase
|
12
29
|
# setup / teardown for servers
|
13
30
|
def setup
|
14
31
|
Fluent::Test.setup
|
@@ -18,12 +35,7 @@ class HTTPOutputTestBase < Test::Unit::TestCase
|
|
18
35
|
@requests = 0
|
19
36
|
@auth = false
|
20
37
|
@dummy_server_thread = Thread.new do
|
21
|
-
srv =
|
22
|
-
WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
|
23
|
-
else
|
24
|
-
logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
|
25
|
-
WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
|
26
|
-
end
|
38
|
+
srv = WEBrick::HTTPServer.new(self.class.server_config)
|
27
39
|
begin
|
28
40
|
allowed_methods = %w(POST PUT)
|
29
41
|
srv.mount_proc('/api/') { |req,res|
|
@@ -71,7 +83,8 @@ class HTTPOutputTestBase < Test::Unit::TestCase
|
|
71
83
|
connected = false
|
72
84
|
while not connected
|
73
85
|
begin
|
74
|
-
|
86
|
+
client = self.class.test_http_client
|
87
|
+
client.request_get('/')
|
75
88
|
connected = true
|
76
89
|
rescue Errno::ECONNREFUSED
|
77
90
|
sleep 0.1
|
@@ -81,20 +94,19 @@ class HTTPOutputTestBase < Test::Unit::TestCase
|
|
81
94
|
end
|
82
95
|
end
|
83
96
|
cv.signal
|
84
|
-
}
|
97
|
+
}
|
85
98
|
mutex = Mutex.new
|
86
99
|
mutex.synchronize {
|
87
100
|
cv.wait(mutex)
|
88
|
-
}
|
101
|
+
}
|
89
102
|
end
|
90
103
|
|
91
104
|
def test_dummy_server
|
92
|
-
|
93
|
-
|
94
|
-
client = Net::HTTP.start(host, port)
|
105
|
+
client = self.class.test_http_client
|
106
|
+
post_header = { 'Content-Type' => 'application/x-www-form-urlencoded' }
|
95
107
|
|
96
108
|
assert_equal '200', client.request_get('/').code
|
97
|
-
assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
|
109
|
+
assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge', post_header).code
|
98
110
|
|
99
111
|
assert_equal 1, @posts.size
|
100
112
|
|
@@ -104,11 +116,11 @@ class HTTPOutputTestBase < Test::Unit::TestCase
|
|
104
116
|
|
105
117
|
@auth = true
|
106
118
|
|
107
|
-
assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
|
119
|
+
assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge', post_header).code
|
108
120
|
|
109
121
|
req_with_auth = lambda do |number, mode, user, pass|
|
110
|
-
|
111
|
-
req =
|
122
|
+
req = Net::HTTP::Post.new("/api/service/metrics/pos")
|
123
|
+
req.content_type = 'application/x-www-form-urlencoded'
|
112
124
|
req.basic_auth user, pass
|
113
125
|
req.set_form_data({'number'=>number, 'mode'=>mode})
|
114
126
|
req
|
@@ -130,55 +142,55 @@ class HTTPOutputTestBase < Test::Unit::TestCase
|
|
130
142
|
@dummy_server_thread.kill
|
131
143
|
@dummy_server_thread.join
|
132
144
|
end
|
145
|
+
|
146
|
+
def create_driver(conf, tag='test.metrics')
|
147
|
+
Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
|
148
|
+
end
|
133
149
|
end
|
134
150
|
|
135
151
|
class HTTPOutputTest < HTTPOutputTestBase
|
136
152
|
CONFIG = %[
|
137
|
-
endpoint_url http://127.0.0.1:#{
|
153
|
+
endpoint_url http://127.0.0.1:#{port}/api/
|
138
154
|
]
|
139
155
|
|
140
156
|
CONFIG_JSON = %[
|
141
|
-
endpoint_url http://127.0.0.1:#{
|
157
|
+
endpoint_url http://127.0.0.1:#{port}/api/
|
142
158
|
serializer json
|
143
159
|
]
|
144
160
|
|
145
161
|
CONFIG_PUT = %[
|
146
|
-
endpoint_url http://127.0.0.1:#{
|
162
|
+
endpoint_url http://127.0.0.1:#{port}/api/
|
147
163
|
http_method put
|
148
164
|
]
|
149
165
|
|
150
166
|
CONFIG_HTTP_ERROR = %[
|
151
|
-
endpoint_url https://127.0.0.1:#{
|
167
|
+
endpoint_url https://127.0.0.1:#{port - 1}/api/
|
152
168
|
]
|
153
169
|
|
154
170
|
CONFIG_HTTP_ERROR_SUPPRESSED = %[
|
155
|
-
endpoint_url https://127.0.0.1:#{
|
171
|
+
endpoint_url https://127.0.0.1:#{port - 1}/api/
|
156
172
|
raise_on_error false
|
157
173
|
]
|
158
174
|
|
159
175
|
RATE_LIMIT_MSEC = 1200
|
160
176
|
|
161
177
|
CONFIG_RATE_LIMIT = %[
|
162
|
-
endpoint_url http://127.0.0.1:#{
|
178
|
+
endpoint_url http://127.0.0.1:#{port}/api/
|
163
179
|
rate_limit_msec #{RATE_LIMIT_MSEC}
|
164
180
|
]
|
165
181
|
|
166
|
-
def create_driver(conf=CONFIG, tag='test.metrics')
|
167
|
-
Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
|
168
|
-
end
|
169
|
-
|
170
182
|
def test_configure
|
171
|
-
d = create_driver
|
172
|
-
assert_equal "http://127.0.0.1:#{
|
183
|
+
d = create_driver CONFIG
|
184
|
+
assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url
|
173
185
|
assert_equal :form, d.instance.serializer
|
174
186
|
|
175
187
|
d = create_driver CONFIG_JSON
|
176
|
-
assert_equal "http://127.0.0.1:#{
|
188
|
+
assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url
|
177
189
|
assert_equal :json, d.instance.serializer
|
178
190
|
end
|
179
191
|
|
180
192
|
def test_emit_form
|
181
|
-
d = create_driver
|
193
|
+
d = create_driver CONFIG
|
182
194
|
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
|
183
195
|
d.run
|
184
196
|
|
@@ -218,7 +230,7 @@ class HTTPOutputTest < HTTPOutputTestBase
|
|
218
230
|
end
|
219
231
|
|
220
232
|
def test_emit_json
|
221
|
-
binary_string = "\xe3\x81\x82"
|
233
|
+
binary_string = "\xe3\x81\x82"
|
222
234
|
d = create_driver CONFIG_JSON
|
223
235
|
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
|
224
236
|
d.run
|
@@ -268,7 +280,7 @@ class HTTPOutputTest < HTTPOutputTestBase
|
|
268
280
|
|
269
281
|
wait_msec = 500
|
270
282
|
sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
|
271
|
-
|
283
|
+
|
272
284
|
assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
|
273
285
|
d.emit(record)
|
274
286
|
d.run
|
@@ -278,7 +290,7 @@ class HTTPOutputTest < HTTPOutputTestBase
|
|
278
290
|
def _current_msec
|
279
291
|
Time.now.to_f * 1000
|
280
292
|
end
|
281
|
-
|
293
|
+
|
282
294
|
def test_auth
|
283
295
|
@auth = true # enable authentication of dummy server
|
284
296
|
|
@@ -313,3 +325,61 @@ class HTTPOutputTest < HTTPOutputTestBase
|
|
313
325
|
end
|
314
326
|
|
315
327
|
end
|
328
|
+
|
329
|
+
class HTTPSOutputTest < HTTPOutputTestBase
|
330
|
+
def self.port
|
331
|
+
5127
|
332
|
+
end
|
333
|
+
|
334
|
+
def self.server_config
|
335
|
+
config = super
|
336
|
+
config[:SSLEnable] = true
|
337
|
+
config[:SSLCertName] = [["CN", WEBrick::Utils::getservername]]
|
338
|
+
config
|
339
|
+
end
|
340
|
+
|
341
|
+
def self.test_http_client
|
342
|
+
super(
|
343
|
+
use_ssl: true,
|
344
|
+
verify_mode: OpenSSL::SSL::VERIFY_NONE,
|
345
|
+
)
|
346
|
+
end
|
347
|
+
|
348
|
+
def test_configure
|
349
|
+
test_uri = URI.parse("https://127.0.0.1/")
|
350
|
+
|
351
|
+
ssl_config = %[
|
352
|
+
endpoint_url https://127.0.0.1:#{self.class.port}/api/
|
353
|
+
]
|
354
|
+
d = create_driver ssl_config
|
355
|
+
expected_endpoint_url = "https://127.0.0.1:#{self.class.port}/api/"
|
356
|
+
assert_equal expected_endpoint_url, d.instance.endpoint_url
|
357
|
+
http_opts = d.instance.http_opts(test_uri)
|
358
|
+
assert_equal true, http_opts[:use_ssl]
|
359
|
+
assert_equal OpenSSL::SSL::VERIFY_PEER, http_opts[:verify_mode]
|
360
|
+
|
361
|
+
no_verify_config = %[
|
362
|
+
endpoint_url https://127.0.0.1:#{self.class.port}/api/
|
363
|
+
ssl_no_verify true
|
364
|
+
]
|
365
|
+
d = create_driver no_verify_config
|
366
|
+
http_opts = d.instance.http_opts(test_uri)
|
367
|
+
assert_equal true, http_opts[:use_ssl]
|
368
|
+
assert_equal OpenSSL::SSL::VERIFY_NONE, http_opts[:verify_mode]
|
369
|
+
end
|
370
|
+
|
371
|
+
def test_emit_form_ssl
|
372
|
+
config = %[
|
373
|
+
endpoint_url https://127.0.0.1:#{self.class.port}/api/
|
374
|
+
ssl_no_verify true
|
375
|
+
]
|
376
|
+
d = create_driver config
|
377
|
+
d.emit({ 'field1' => 50 })
|
378
|
+
d.run
|
379
|
+
|
380
|
+
assert_equal 1, @posts.size
|
381
|
+
record = @posts[0]
|
382
|
+
|
383
|
+
assert_equal '50', record[:form]['field1']
|
384
|
+
end
|
385
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-out-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marica Odagaki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: yajl-ruby
|
@@ -116,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
116
116
|
requirements:
|
117
117
|
- - ">="
|
118
118
|
- !ruby/object:Gem::Version
|
119
|
-
version:
|
119
|
+
version: 2.1.0
|
120
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - ">="
|
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
124
124
|
version: '0'
|
125
125
|
requirements: []
|
126
126
|
rubyforge_project:
|
127
|
-
rubygems_version: 2.
|
127
|
+
rubygems_version: 2.6.8
|
128
128
|
signing_key:
|
129
129
|
specification_version: 4
|
130
130
|
summary: A generic Fluentd output plugin to send logs to an HTTP endpoint
|