fluent-plugin-out-http-ext 0.1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.travis.yml +21 -0
- data/CHANGELOG.md +24 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +11 -0
- data/README.md +36 -0
- data/Rakefile +11 -0
- data/fluent-plugin-out-http-ext.gemspec +22 -0
- data/lib/fluent/plugin/out_http_ext.rb +186 -0
- data/lib/fluent/test/http_output_test.rb +54 -0
- data/test/plugin/test_out_http_ext.rb +315 -0
- metadata +134 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 15c119a9a2f65d9b83eafeb4d3be3db265bfe317
|
4
|
+
data.tar.gz: 684d0ca37de1c5b41cac5747d9cef90ea7fd1fda
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9500eb598450b09855a51e6e2c1ded0f78b6105c1828dc11378c8c947637da156fa14ef6c2428cb460da5a773f887907048cbfee074817bfb4ea0c22f2740276
|
7
|
+
data.tar.gz: 69810ae41774022dd952b95f8bfda891ee2c257de1b76535dcfb13255b48da33df6628f484022ab3ca8798577aadbe8e6eca83152b2225bff5ce319c77c68cda
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## 0.1.5
|
4
|
+
* Add headers directive
|
5
|
+
* Add use_ssl directive
|
6
|
+
|
7
|
+
## 0.1.4
|
8
|
+
* #11 Updated Fluentd dependency to: [">= 0.10.0", "< 2"]
|
9
|
+
* #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)
|
10
|
+
|
11
|
+
## 0.1.3
|
12
|
+
* Added a new configuration option: `raise_on_error` (default: true)
|
13
|
+
* In order to let the plugin raise exceptions like it did in 0.1.1: keep using your configuration as-is
|
14
|
+
* In order to suppress all exceptions: add `raise_on_error false` to your configuration
|
15
|
+
|
16
|
+
## 0.1.2
|
17
|
+
* #6 Catch all `StandardError`s during HTTP request to prevent td-agent from freezing
|
18
|
+
|
19
|
+
## 0.1.1
|
20
|
+
* #2 Use yajl instead of json as json serializer
|
21
|
+
* #1 Fix a bug where a nil HTTP response caused the plugin to stop working
|
22
|
+
|
23
|
+
## 0.1.0
|
24
|
+
* Initial release
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
2
|
+
you may not use this file except in compliance with the License.
|
3
|
+
You may obtain a copy of the License at
|
4
|
+
|
5
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
6
|
+
|
7
|
+
Unless required by applicable law or agreed to in writing, software
|
8
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
9
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
10
|
+
See the License for the specific language governing permissions and
|
11
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# fluent-plugin-out-http-ext, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
**This is a fork of [ento / fluent-plugin-out-http](https://github.com/ento/fluent-plugin-out-http)**
|
4
|
+
|
5
|
+
A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint
|
6
|
+
|
7
|
+
## Configuration options
|
8
|
+
|
9
|
+
<match *>
|
10
|
+
type http-ext
|
11
|
+
endpoint_url http://localhost.local/api/<data.id> # <data.id> refres to data.id in the record like {"data"=> {"id"=> 1, "name"=> "foo"}}
|
12
|
+
http_method put # default: post
|
13
|
+
serializer json # default: form
|
14
|
+
rate_limit_msec 100 # default: 0 = no rate limiting
|
15
|
+
raise_on_error false # default: true
|
16
|
+
authentication basic # default: none
|
17
|
+
username alice # default: ''
|
18
|
+
password bobpop # default: '', secret: true
|
19
|
+
use_ssl true # default: false
|
20
|
+
<headers>
|
21
|
+
HeaderExample1 header1
|
22
|
+
HeaderExample2 header2
|
23
|
+
</headers>
|
24
|
+
</match>
|
25
|
+
|
26
|
+
## Usage notes
|
27
|
+
|
28
|
+
If you'd like to retry failed requests, consider using [fluent-plugin-bufferize][3].
|
29
|
+
|
30
|
+
----
|
31
|
+
|
32
|
+
Heavily based on [fluent-plugin-growthforecast][2]
|
33
|
+
|
34
|
+
[1]: http://fluentd.org/
|
35
|
+
[2]: https://github.com/tagomoris/fluent-plugin-growthforecast
|
36
|
+
[3]: https://github.com/sabottenda/fluent-plugin-bufferize
|
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-out-http-ext"
|
5
|
+
gem.version = "0.1.5.3"
|
6
|
+
gem.authors = ["Toshiya Kawasaki"]
|
7
|
+
gem.email = ["kawasakitoshiya@gmail.com"]
|
8
|
+
gem.summary = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL and Header option}
|
9
|
+
gem.description = gem.summary
|
10
|
+
gem.licenses = ["Apache-2.0"]
|
11
|
+
|
12
|
+
gem.files = `git ls-files`.split($\)
|
13
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
14
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
|
17
|
+
gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
|
18
|
+
gem.add_runtime_dependency "fluentd", [">= 0.10.0", "< 2"]
|
19
|
+
gem.add_development_dependency "bundler"
|
20
|
+
gem.add_development_dependency "rake"
|
21
|
+
gem.add_development_dependency "test-unit", ">= 3.1.0"
|
22
|
+
end
|
@@ -0,0 +1,186 @@
|
|
1
|
+
class Hash
|
2
|
+
"""
|
3
|
+
each traverse in hash
|
4
|
+
"""
|
5
|
+
def each_deep(&proc)
|
6
|
+
self.each_deep_detail([], &proc)
|
7
|
+
end
|
8
|
+
|
9
|
+
def each_deep_detail(directory, &proc)
|
10
|
+
self.each do |k, v|
|
11
|
+
current = directory + [k]
|
12
|
+
if v.kind_of?(v.class)
|
13
|
+
v.each_deep_detail(current, &proc)
|
14
|
+
else
|
15
|
+
yield(current, v)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
class Fluent::HTTPOutput < Fluent::Output
|
23
|
+
Fluent::Plugin.register_output('http_ext', self)
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
super
|
27
|
+
require 'net/http'
|
28
|
+
require 'uri'
|
29
|
+
require 'yajl'
|
30
|
+
end
|
31
|
+
|
32
|
+
# Endpoint URL ex. localhost.local/api/
|
33
|
+
config_param :endpoint_url, :string
|
34
|
+
|
35
|
+
# HTTP method
|
36
|
+
config_param :http_method, :string, :default => :post
|
37
|
+
|
38
|
+
# form | json
|
39
|
+
config_param :serializer, :string, :default => :form
|
40
|
+
|
41
|
+
# true | false
|
42
|
+
config_param :use_ssl, :bool, :default => false
|
43
|
+
|
44
|
+
# Simple rate limiting: ignore any records within `rate_limit_msec`
|
45
|
+
# since the last one.
|
46
|
+
config_param :rate_limit_msec, :integer, :default => 0
|
47
|
+
|
48
|
+
# Raise errors that were rescued during HTTP requests?
|
49
|
+
config_param :raise_on_error, :bool, :default => true
|
50
|
+
|
51
|
+
|
52
|
+
# nil | 'none' | 'basic'
|
53
|
+
config_param :authentication, :string, :default => nil
|
54
|
+
config_param :username, :string, :default => ''
|
55
|
+
config_param :password, :string, :default => '', :secret => true
|
56
|
+
|
57
|
+
def configure(conf)
|
58
|
+
super
|
59
|
+
|
60
|
+
serializers = [:json, :form]
|
61
|
+
@serializer = if serializers.include? @serializer.intern
|
62
|
+
@serializer.intern
|
63
|
+
else
|
64
|
+
:form
|
65
|
+
end
|
66
|
+
|
67
|
+
http_methods = [:get, :put, :post, :delete]
|
68
|
+
@http_method = if http_methods.include? @http_method.intern
|
69
|
+
@http_method.intern
|
70
|
+
else
|
71
|
+
:post
|
72
|
+
end
|
73
|
+
|
74
|
+
@auth = case @authentication
|
75
|
+
when 'basic' then :basic
|
76
|
+
else
|
77
|
+
:none
|
78
|
+
end
|
79
|
+
@headers = {}
|
80
|
+
conf.elements.each do |element|
|
81
|
+
if element.name == 'headers'
|
82
|
+
@headers = element.to_hash
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def start
|
88
|
+
super
|
89
|
+
end
|
90
|
+
|
91
|
+
def shutdown
|
92
|
+
super
|
93
|
+
end
|
94
|
+
|
95
|
+
def format_url(tag, time, record)
|
96
|
+
'''
|
97
|
+
replace format string to value
|
98
|
+
example
|
99
|
+
/test/<data> =(use {data: 1})> /test/1
|
100
|
+
/test/<hash.data> =(use {hash:{data:2}})> /test/2
|
101
|
+
'''
|
102
|
+
result_url = @endpoint_url
|
103
|
+
record.each_deep do |key_dir, value|
|
104
|
+
result_url = result_url.gsub(/<#{key_dir.join(".")}>/, value.to_s)
|
105
|
+
end
|
106
|
+
return result_url
|
107
|
+
end
|
108
|
+
|
109
|
+
def set_body(req, tag, time, record)
|
110
|
+
if @serializer == :json
|
111
|
+
set_json_body(req, record)
|
112
|
+
else
|
113
|
+
req.set_form_data(record)
|
114
|
+
end
|
115
|
+
req
|
116
|
+
end
|
117
|
+
|
118
|
+
def set_header(req, tag, time, record)
|
119
|
+
@headers.each do |key, value|
|
120
|
+
req[key] = value
|
121
|
+
end
|
122
|
+
req
|
123
|
+
end
|
124
|
+
|
125
|
+
def set_json_body(req, data)
|
126
|
+
req.body = Yajl.dump(data)
|
127
|
+
req['Content-Type'] = 'application/json'
|
128
|
+
end
|
129
|
+
|
130
|
+
def create_request(tag, time, record)
|
131
|
+
url = format_url(tag, time, record)
|
132
|
+
uri = URI.parse(url)
|
133
|
+
req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
|
134
|
+
set_body(req, tag, time, record)
|
135
|
+
set_header(req, tag, time, record)
|
136
|
+
return req, uri
|
137
|
+
end
|
138
|
+
|
139
|
+
def send_request(req, uri)
|
140
|
+
is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
|
141
|
+
if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
|
142
|
+
$log.info('Dropped request due to rate limiting')
|
143
|
+
return
|
144
|
+
end
|
145
|
+
|
146
|
+
res = nil
|
147
|
+
|
148
|
+
begin
|
149
|
+
if @auth and @auth == :basic
|
150
|
+
req.basic_auth(@username, @password)
|
151
|
+
end
|
152
|
+
@last_request_time = Time.now.to_f
|
153
|
+
client = Net::HTTP.new(uri.host, uri.port)
|
154
|
+
if @use_ssl
|
155
|
+
client.use_ssl = true
|
156
|
+
client.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE
|
157
|
+
end
|
158
|
+
res = client.start {|http| http.request(req) }
|
159
|
+
rescue => e # rescue all StandardErrors
|
160
|
+
# server didn't respond
|
161
|
+
$log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
|
162
|
+
raise e if @raise_on_error
|
163
|
+
else
|
164
|
+
unless res and res.is_a?(Net::HTTPSuccess)
|
165
|
+
res_summary = if res
|
166
|
+
"#{res.code} #{res.message} #{res.body}"
|
167
|
+
else
|
168
|
+
"res=nil"
|
169
|
+
end
|
170
|
+
$log.warn "failed to #{req.method} #{uri} (#{res_summary})"
|
171
|
+
end #end unless
|
172
|
+
end # end begin
|
173
|
+
end # end send_request
|
174
|
+
|
175
|
+
def handle_record(tag, time, record)
|
176
|
+
req, uri = create_request(tag, time, record)
|
177
|
+
send_request(req, uri)
|
178
|
+
end
|
179
|
+
|
180
|
+
def emit(tag, es, chain)
|
181
|
+
es.each do |time, record|
|
182
|
+
handle_record(tag, time, record)
|
183
|
+
end
|
184
|
+
chain.next
|
185
|
+
end
|
186
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
|
4
|
+
begin
|
5
|
+
Bundler.setup(:default, :development)
|
6
|
+
rescue Bundler::BundlerError => e
|
7
|
+
$stderr.puts e.message
|
8
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
9
|
+
exit e.status_code
|
10
|
+
end
|
11
|
+
|
12
|
+
require 'test/unit'
|
13
|
+
|
14
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
15
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
16
|
+
|
17
|
+
require 'fluent/test'
|
18
|
+
|
19
|
+
unless ENV.has_key?('VERBOSE')
|
20
|
+
nulllogger = Object.new
|
21
|
+
nulllogger.instance_eval {|obj|
|
22
|
+
def method_missing(method, *args)
|
23
|
+
# pass
|
24
|
+
end
|
25
|
+
}
|
26
|
+
$log = nulllogger
|
27
|
+
end
|
28
|
+
|
29
|
+
class Test::Unit::TestCase
|
30
|
+
end
|
31
|
+
|
32
|
+
require 'webrick'
|
33
|
+
|
34
|
+
# to handle POST/PUT/DELETE ...
|
35
|
+
module WEBrick::HTTPServlet
|
36
|
+
class ProcHandler < AbstractServlet
|
37
|
+
alias do_POST do_GET
|
38
|
+
alias do_PUT do_GET
|
39
|
+
alias do_DELETE do_GET
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_code(server, port, path, headers={})
|
44
|
+
require 'net/http'
|
45
|
+
Net::HTTP.start(server, port){|http|
|
46
|
+
http.get(path, headers).code
|
47
|
+
}
|
48
|
+
end
|
49
|
+
def get_content(server, port, path, headers={})
|
50
|
+
require 'net/http'
|
51
|
+
Net::HTTP.start(server, port){|http|
|
52
|
+
http.get(path, headers).body
|
53
|
+
}
|
54
|
+
end
|
@@ -0,0 +1,315 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'uri'
|
3
|
+
require 'yajl'
|
4
|
+
require 'fluent/test/http_output_test'
|
5
|
+
require 'fluent/plugin/out_http_ext'
|
6
|
+
|
7
|
+
|
8
|
+
TEST_LISTEN_PORT = 5126
|
9
|
+
|
10
|
+
|
11
|
+
class HTTPOutputTestBase < Test::Unit::TestCase
|
12
|
+
# setup / teardown for servers
|
13
|
+
def setup
|
14
|
+
Fluent::Test.setup
|
15
|
+
@posts = []
|
16
|
+
@puts = []
|
17
|
+
@prohibited = 0
|
18
|
+
@requests = 0
|
19
|
+
@auth = false
|
20
|
+
@dummy_server_thread = Thread.new do
|
21
|
+
srv = if ENV['VERBOSE']
|
22
|
+
WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
|
23
|
+
else
|
24
|
+
logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
|
25
|
+
WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
|
26
|
+
end
|
27
|
+
begin
|
28
|
+
allowed_methods = %w(POST PUT)
|
29
|
+
srv.mount_proc('/api/') { |req,res|
|
30
|
+
@requests += 1
|
31
|
+
unless allowed_methods.include? req.request_method
|
32
|
+
res.status = 405
|
33
|
+
res.body = 'request method mismatch'
|
34
|
+
next
|
35
|
+
end
|
36
|
+
if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
|
37
|
+
# ok, authorized
|
38
|
+
elsif @auth
|
39
|
+
res.status = 403
|
40
|
+
@prohibited += 1
|
41
|
+
next
|
42
|
+
else
|
43
|
+
# ok, authorization not required
|
44
|
+
end
|
45
|
+
|
46
|
+
record = {:auth => nil}
|
47
|
+
if req.content_type == 'application/json'
|
48
|
+
record[:json] = Yajl.load(req.body)
|
49
|
+
else
|
50
|
+
record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
|
51
|
+
end
|
52
|
+
|
53
|
+
instance_variable_get("@#{req.request_method.downcase}s").push(record)
|
54
|
+
|
55
|
+
res.status = 200
|
56
|
+
}
|
57
|
+
srv.mount_proc('/') { |req,res|
|
58
|
+
res.status = 200
|
59
|
+
res.body = 'running'
|
60
|
+
}
|
61
|
+
srv.start
|
62
|
+
ensure
|
63
|
+
srv.shutdown
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# to wait completion of dummy server.start()
|
68
|
+
require 'thread'
|
69
|
+
cv = ConditionVariable.new
|
70
|
+
watcher = Thread.new {
|
71
|
+
connected = false
|
72
|
+
while not connected
|
73
|
+
begin
|
74
|
+
get_content('localhost', TEST_LISTEN_PORT, '/')
|
75
|
+
connected = true
|
76
|
+
rescue Errno::ECONNREFUSED
|
77
|
+
sleep 0.1
|
78
|
+
rescue StandardError => e
|
79
|
+
p e
|
80
|
+
sleep 0.1
|
81
|
+
end
|
82
|
+
end
|
83
|
+
cv.signal
|
84
|
+
}
|
85
|
+
mutex = Mutex.new
|
86
|
+
mutex.synchronize {
|
87
|
+
cv.wait(mutex)
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_dummy_server
|
92
|
+
host = '127.0.0.1'
|
93
|
+
port = TEST_LISTEN_PORT
|
94
|
+
client = Net::HTTP.start(host, port)
|
95
|
+
|
96
|
+
assert_equal '200', client.request_get('/').code
|
97
|
+
assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
|
98
|
+
|
99
|
+
assert_equal 1, @posts.size
|
100
|
+
|
101
|
+
assert_equal '1', @posts[0][:form]['number']
|
102
|
+
assert_equal 'gauge', @posts[0][:form]['mode']
|
103
|
+
assert_nil @posts[0][:auth]
|
104
|
+
|
105
|
+
@auth = true
|
106
|
+
|
107
|
+
assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
|
108
|
+
|
109
|
+
req_with_auth = lambda do |number, mode, user, pass|
|
110
|
+
url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
|
111
|
+
req = Net::HTTP::Post.new(url.path)
|
112
|
+
req.basic_auth user, pass
|
113
|
+
req.set_form_data({'number'=>number, 'mode'=>mode})
|
114
|
+
req
|
115
|
+
end
|
116
|
+
|
117
|
+
assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
|
118
|
+
|
119
|
+
assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
|
120
|
+
|
121
|
+
assert_equal 1, @posts.size
|
122
|
+
|
123
|
+
assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
|
124
|
+
|
125
|
+
assert_equal 2, @posts.size
|
126
|
+
|
127
|
+
end
|
128
|
+
|
129
|
+
def teardown
|
130
|
+
@dummy_server_thread.kill
|
131
|
+
@dummy_server_thread.join
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
class HTTPOutputTest < HTTPOutputTestBase
|
136
|
+
CONFIG = %[
|
137
|
+
endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
|
138
|
+
]
|
139
|
+
|
140
|
+
CONFIG_JSON = %[
|
141
|
+
endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
|
142
|
+
serializer json
|
143
|
+
]
|
144
|
+
|
145
|
+
CONFIG_PUT = %[
|
146
|
+
endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
|
147
|
+
http_method put
|
148
|
+
]
|
149
|
+
|
150
|
+
CONFIG_HTTP_ERROR = %[
|
151
|
+
endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
|
152
|
+
]
|
153
|
+
|
154
|
+
CONFIG_HTTP_ERROR_SUPPRESSED = %[
|
155
|
+
endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
|
156
|
+
raise_on_error false
|
157
|
+
]
|
158
|
+
|
159
|
+
RATE_LIMIT_MSEC = 1200
|
160
|
+
|
161
|
+
CONFIG_RATE_LIMIT = %[
|
162
|
+
endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
|
163
|
+
rate_limit_msec #{RATE_LIMIT_MSEC}
|
164
|
+
]
|
165
|
+
|
166
|
+
def create_driver(conf=CONFIG, tag='test.metrics')
|
167
|
+
Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
|
168
|
+
end
|
169
|
+
|
170
|
+
def test_configure
|
171
|
+
d = create_driver
|
172
|
+
assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
|
173
|
+
assert_equal :form, d.instance.serializer
|
174
|
+
|
175
|
+
d = create_driver CONFIG_JSON
|
176
|
+
assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
|
177
|
+
assert_equal :json, d.instance.serializer
|
178
|
+
end
|
179
|
+
|
180
|
+
def test_emit_form
|
181
|
+
d = create_driver
|
182
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
|
183
|
+
d.run
|
184
|
+
|
185
|
+
assert_equal 1, @posts.size
|
186
|
+
record = @posts[0]
|
187
|
+
|
188
|
+
assert_equal '50', record[:form]['field1']
|
189
|
+
assert_equal '20', record[:form]['field2']
|
190
|
+
assert_equal '10', record[:form]['field3']
|
191
|
+
assert_equal '1', record[:form]['otherfield']
|
192
|
+
assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase
|
193
|
+
assert_nil record[:auth]
|
194
|
+
|
195
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
|
196
|
+
d.run
|
197
|
+
|
198
|
+
assert_equal 2, @posts.size
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_emit_form_put
|
202
|
+
d = create_driver CONFIG_PUT
|
203
|
+
d.emit({ 'field1' => 50 })
|
204
|
+
d.run
|
205
|
+
|
206
|
+
assert_equal 0, @posts.size
|
207
|
+
assert_equal 1, @puts.size
|
208
|
+
record = @puts[0]
|
209
|
+
|
210
|
+
assert_equal '50', record[:form]['field1']
|
211
|
+
assert_nil record[:auth]
|
212
|
+
|
213
|
+
d.emit({ 'field1' => 50 })
|
214
|
+
d.run
|
215
|
+
|
216
|
+
assert_equal 0, @posts.size
|
217
|
+
assert_equal 2, @puts.size
|
218
|
+
end
|
219
|
+
|
220
|
+
def test_emit_json
|
221
|
+
binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
|
222
|
+
d = create_driver CONFIG_JSON
|
223
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
|
224
|
+
d.run
|
225
|
+
|
226
|
+
assert_equal 1, @posts.size
|
227
|
+
record = @posts[0]
|
228
|
+
|
229
|
+
assert_equal 50, record[:json]['field1']
|
230
|
+
assert_equal 20, record[:json]['field2']
|
231
|
+
assert_equal 10, record[:json]['field3']
|
232
|
+
assert_equal 1, record[:json]['otherfield']
|
233
|
+
assert_equal binary_string, record[:json]['binary']
|
234
|
+
assert_nil record[:auth]
|
235
|
+
end
|
236
|
+
|
237
|
+
def test_http_error_is_raised
|
238
|
+
d = create_driver CONFIG_HTTP_ERROR
|
239
|
+
assert_raise Errno::ECONNREFUSED do
|
240
|
+
d.emit({ 'field1' => 50 })
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def test_http_error_is_suppressed_with_raise_on_error_false
|
245
|
+
d = create_driver CONFIG_HTTP_ERROR_SUPPRESSED
|
246
|
+
d.emit({ 'field1' => 50 })
|
247
|
+
d.run
|
248
|
+
# drive asserts the next output chain is called;
|
249
|
+
# so no exception means our plugin handled the error
|
250
|
+
|
251
|
+
assert_equal 0, @requests
|
252
|
+
end
|
253
|
+
|
254
|
+
def test_rate_limiting
|
255
|
+
d = create_driver CONFIG_RATE_LIMIT
|
256
|
+
record = { :k => 1 }
|
257
|
+
|
258
|
+
last_emit = _current_msec
|
259
|
+
d.emit(record)
|
260
|
+
d.run
|
261
|
+
|
262
|
+
assert_equal 1, @posts.size
|
263
|
+
|
264
|
+
d.emit({})
|
265
|
+
d.run
|
266
|
+
assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
|
267
|
+
assert_equal 1, @posts.size
|
268
|
+
|
269
|
+
wait_msec = 500
|
270
|
+
sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
|
271
|
+
|
272
|
+
assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
|
273
|
+
d.emit(record)
|
274
|
+
d.run
|
275
|
+
assert_equal 2, @posts.size
|
276
|
+
end
|
277
|
+
|
278
|
+
def _current_msec
|
279
|
+
Time.now.to_f * 1000
|
280
|
+
end
|
281
|
+
|
282
|
+
def test_auth
|
283
|
+
@auth = true # enable authentication of dummy server
|
284
|
+
|
285
|
+
d = create_driver(CONFIG, 'test.metrics')
|
286
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
|
287
|
+
d.run # failed in background, and output warn log
|
288
|
+
|
289
|
+
assert_equal 0, @posts.size
|
290
|
+
assert_equal 1, @prohibited
|
291
|
+
|
292
|
+
d = create_driver(CONFIG + %[
|
293
|
+
authentication basic
|
294
|
+
username alice
|
295
|
+
password wrong_password
|
296
|
+
], 'test.metrics')
|
297
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
|
298
|
+
d.run # failed in background, and output warn log
|
299
|
+
|
300
|
+
assert_equal 0, @posts.size
|
301
|
+
assert_equal 2, @prohibited
|
302
|
+
|
303
|
+
d = create_driver(CONFIG + %[
|
304
|
+
authentication basic
|
305
|
+
username alice
|
306
|
+
password secret!
|
307
|
+
], 'test.metrics')
|
308
|
+
d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
|
309
|
+
d.run # failed in background, and output warn log
|
310
|
+
|
311
|
+
assert_equal 1, @posts.size
|
312
|
+
assert_equal 2, @prohibited
|
313
|
+
end
|
314
|
+
|
315
|
+
end
|
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-out-http-ext
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.5.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Toshiya Kawasaki
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: yajl-ruby
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.10.0
|
34
|
+
- - "<"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '2'
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 0.10.0
|
44
|
+
- - "<"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '2'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bundler
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: rake
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: test-unit
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 3.1.0
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 3.1.0
|
89
|
+
description: A generic Fluentd output plugin to send logs to an HTTP endpoint with
|
90
|
+
SSL and Header option
|
91
|
+
email:
|
92
|
+
- kawasakitoshiya@gmail.com
|
93
|
+
executables: []
|
94
|
+
extensions: []
|
95
|
+
extra_rdoc_files: []
|
96
|
+
files:
|
97
|
+
- ".gitignore"
|
98
|
+
- ".travis.yml"
|
99
|
+
- CHANGELOG.md
|
100
|
+
- Gemfile
|
101
|
+
- LICENSE.txt
|
102
|
+
- README.md
|
103
|
+
- Rakefile
|
104
|
+
- fluent-plugin-out-http-ext.gemspec
|
105
|
+
- lib/fluent/plugin/out_http_ext.rb
|
106
|
+
- lib/fluent/test/http_output_test.rb
|
107
|
+
- test/plugin/test_out_http_ext.rb
|
108
|
+
homepage:
|
109
|
+
licenses:
|
110
|
+
- Apache-2.0
|
111
|
+
metadata: {}
|
112
|
+
post_install_message:
|
113
|
+
rdoc_options: []
|
114
|
+
require_paths:
|
115
|
+
- lib
|
116
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - ">="
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
121
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
requirements: []
|
127
|
+
rubyforge_project:
|
128
|
+
rubygems_version: 2.4.6
|
129
|
+
signing_key:
|
130
|
+
specification_version: 4
|
131
|
+
summary: A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL
|
132
|
+
and Header option
|
133
|
+
test_files:
|
134
|
+
- test/plugin/test_out_http_ext.rb
|