fluent-plugin-out-http-buffered 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/Gemfile +3 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +36 -0
- data/Rakefile +11 -0
- data/VERSION +1 -0
- data/fluent-plugin-out-http-buffered.gemspec +39 -0
- data/lib/fluent/plugin/out_http_buffered.rb +109 -0
- data/test/fluent/plugin/test_out_http_buffered.rb +97 -0
- data/test/helper.rb +36 -0
- metadata +110 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
OWU0ZDllMjIzMTA1MjkxZWVlNDdiMzJlZGZiMGUzYWExNWNiZjEwMg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
OTExYzg3MDQ2YWZiYjQwZTVhOTI1ZGY4ZjVkOGJjMmQ1OTQ3NGU3OA==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
OGY5MDdlMGM5ZGVhZDIxMTU0ZjI4ZWIyYTJhY2Q0YTM0NjUzYTI4ZWU2OGEx
|
10
|
+
Yzg4Yzc1OGEzN2M5ODQ1NWJhMGI4NTcxMWExM2IzOTE3NDVmYTMxNDBmOGRm
|
11
|
+
NTdiY2Y1YTA5YTY1MmVjZGEwMTQ5MmE4MWQ5NjBlMjYxMjJjYWU=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZjcxOThiZWYzNTMxMzUwMTUwYThjOTBlMTg1NmY0NjVlM2I4N2I3ZDU3NmMx
|
14
|
+
MjI2YzVhOTE0ZTdhN2Y4ZjA5MDkwMTM0OTZmNTgwZGI0N2IwMWY2MmEyYzM2
|
15
|
+
Y2JjMDYxNTlkYzE1MzQyYWJiMGYyYTY3OGMxNTdlNjA0YTQ3OWM=
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2013 ablagoev
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
= fluent-out-http-buffered {<img src="https://travis-ci.org/ablagoev/fluent-plugin-out-http-buffered.png?branch=master" />}[https://travis-ci.org/ablagoev/fluent-plugin-out-http-buffered]
|
2
|
+
|
3
|
+
This is an output plugin for (Fluentd)[http://fluentd.org/] which deliveres buffered log messages to an http endpoint.
|
4
|
+
|
5
|
+
It has configurable (read)[http://ruby-doc.org/stdlib-2.0/libdoc/net/http/rdoc/Net/HTTP.html#method-i-read_timeout-3D] and (open)[http://ruby-doc.org/stdlib-2.0/libdoc/net/http/rdoc/Net/HTTP.html#open_timeout] timeouts.
|
6
|
+
|
7
|
+
Clients can also configure which http response statuses should be retried (in most cases clients would want to retry on status 500).
|
8
|
+
|
9
|
+
All messages are sent through POST in json format.
|
10
|
+
|
11
|
+
The plugin was influenced by the standard (http output plugin)[https://github.com/ento/fluent-plugin-out-http].
|
12
|
+
|
13
|
+
== Installation:
|
14
|
+
|
15
|
+
`gem install fluent-out-http-buffered`
|
16
|
+
|
17
|
+
== Usage:
|
18
|
+
|
19
|
+
# Configuration file fluent.conf
|
20
|
+
<match fluentd.test.*>
|
21
|
+
type http_buffered
|
22
|
+
flush_interval 2s
|
23
|
+
#Endpoint for messages
|
24
|
+
endpoint_url http://localhost/fluent.php
|
25
|
+
#Comma separated list of http statuses which need to be retried
|
26
|
+
http_retry_statuses 500, 403
|
27
|
+
#Read timeout in seconds, supports floats
|
28
|
+
http_read_timeout 2.2
|
29
|
+
#Open timeout in seconds, supports floats
|
30
|
+
http_open_timeout 2.34
|
31
|
+
</match>
|
32
|
+
|
33
|
+
== Copyright
|
34
|
+
|
35
|
+
Copyright (c) 2013 ablagoev. See LICENSE.txt for
|
36
|
+
further details.
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'fluent-plugin-out-http-buffered'
|
6
|
+
s.version = File.read("VERSION").strip
|
7
|
+
s.date = '2013-05-13'
|
8
|
+
s.summary = "Fluentd http buffered output plugin"
|
9
|
+
s.description = "Send fluent buffered logs to an http endpoint"
|
10
|
+
s.authors = ["Alexander Blagoev"]
|
11
|
+
s.email = 'alexander.i.blagoev@gmail.com'
|
12
|
+
s.homepage =
|
13
|
+
'http://github.com/ablagoev/fluent-plugin-out-http-buffered'
|
14
|
+
|
15
|
+
s.files = [
|
16
|
+
"lib/fluent/plugin/out_http_buffered.rb",
|
17
|
+
"Gemfile",
|
18
|
+
"LICENSE.txt",
|
19
|
+
"README.rdoc",
|
20
|
+
"Rakefile",
|
21
|
+
"VERSION",
|
22
|
+
"fluent-plugin-out-http-buffered.gemspec",
|
23
|
+
"test/helper.rb",
|
24
|
+
"test/fluent/plugin/test_out_http_buffered.rb",
|
25
|
+
]
|
26
|
+
|
27
|
+
s.extra_rdoc_files = [
|
28
|
+
"LICENSE.txt",
|
29
|
+
"README.rdoc"
|
30
|
+
]
|
31
|
+
s.licenses = ["MIT"]
|
32
|
+
|
33
|
+
s.require_paths = ['lib']
|
34
|
+
|
35
|
+
s.add_dependency "fluentd", "~> 0.10.0"
|
36
|
+
s.add_development_dependency "rake", ">= 0.9.2"
|
37
|
+
s.add_development_dependency "rspec-mocks", ">= 2.13.0"
|
38
|
+
s.add_development_dependency "bundler", ">= 1.3.4"
|
39
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
# Main Output plugin class
|
5
|
+
class HttpBufferedOutput < Fluent::BufferedOutput
|
6
|
+
Fluent::Plugin.register_output('http_buffered', self)
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
require 'net/http'
|
11
|
+
require 'uri'
|
12
|
+
end
|
13
|
+
|
14
|
+
# Endpoint URL ex. localhost.local/api/
|
15
|
+
config_param :endpoint_url, :string
|
16
|
+
|
17
|
+
# statuses under which to retry
|
18
|
+
config_param :http_retry_statuses, :string, default: ''
|
19
|
+
|
20
|
+
# read timeout for the http call
|
21
|
+
config_param :http_read_timeout, :float, default: 2.0
|
22
|
+
|
23
|
+
# open timeout for the http call
|
24
|
+
config_param :http_open_timeout, :float, default: 2.0
|
25
|
+
|
26
|
+
def configure(conf)
|
27
|
+
super
|
28
|
+
|
29
|
+
# Check if endpoint URL is valid
|
30
|
+
unless @endpoint_url =~ /^#{URI.regexp}$/
|
31
|
+
fail Fluent::ConfigError, 'endpoint_url invalid'
|
32
|
+
end
|
33
|
+
|
34
|
+
begin
|
35
|
+
@uri = URI.parse(@endpoint_url)
|
36
|
+
rescue URI::InvalidURIError
|
37
|
+
raise Fluent::ConfigError, 'endpoint_url invalid'
|
38
|
+
end
|
39
|
+
|
40
|
+
# Parse http statuses
|
41
|
+
@statuses = @http_retry_statuses.split(',').map { |status| status.to_i }
|
42
|
+
|
43
|
+
@statuses = [] if @statuses.nil?
|
44
|
+
|
45
|
+
@http = Net::HTTP.new(@uri.host, @uri.port)
|
46
|
+
@http.read_timeout = @http_read_timeout
|
47
|
+
@http.open_timeout = @http_open_timeout
|
48
|
+
end
|
49
|
+
|
50
|
+
def start
|
51
|
+
super
|
52
|
+
end
|
53
|
+
|
54
|
+
def shutdown
|
55
|
+
super
|
56
|
+
begin
|
57
|
+
@http.finish
|
58
|
+
rescue
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def format(tag, time, record)
|
63
|
+
[tag, time, record].to_msgpack
|
64
|
+
end
|
65
|
+
|
66
|
+
def write(chunk)
|
67
|
+
data = []
|
68
|
+
chunk.msgpack_each do |(tag, time, record)|
|
69
|
+
data << [tag, time, record]
|
70
|
+
end
|
71
|
+
|
72
|
+
request = create_request(data)
|
73
|
+
|
74
|
+
begin
|
75
|
+
response = @http.start do |http|
|
76
|
+
request = create_request(data)
|
77
|
+
http.request request
|
78
|
+
end
|
79
|
+
|
80
|
+
if @statuses.include? response.code.to_i
|
81
|
+
# Raise an exception so that fluent retries
|
82
|
+
fail "Server returned bad status: #{response.code}"
|
83
|
+
end
|
84
|
+
rescue IOError, EOFError, SystemCallError => e
|
85
|
+
# server didn't respond
|
86
|
+
$log.warn "Net::HTTP.#{request.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
|
87
|
+
ensure
|
88
|
+
begin
|
89
|
+
@http.finish
|
90
|
+
rescue
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
protected
|
96
|
+
|
97
|
+
def create_request(data)
|
98
|
+
request = Net::HTTP::Post.new(@uri.request_uri)
|
99
|
+
|
100
|
+
# Headers
|
101
|
+
request['Content-Type'] = 'application/json'
|
102
|
+
|
103
|
+
# Body
|
104
|
+
request.body = JSON.dump(data)
|
105
|
+
|
106
|
+
request
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'helper'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
# Test case for the output plugin
|
7
|
+
class HttpBufferedOutputTest < Test::Unit::TestCase
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
end
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
endpoint_url http://local.endpoint
|
14
|
+
]
|
15
|
+
|
16
|
+
# Used to test invalid method config
|
17
|
+
CONFIG_METHOD = %[
|
18
|
+
endpoint_url local.endpoint
|
19
|
+
http_method invalid_method
|
20
|
+
]
|
21
|
+
|
22
|
+
def create_driver(conf = CONFIG)
|
23
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::HttpBufferedOutput).configure(conf)
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_configure
|
27
|
+
d = create_driver
|
28
|
+
assert_equal 'http://local.endpoint', d.instance.instance_eval { @endpoint_url }
|
29
|
+
assert_equal '', d.instance.instance_eval { @http_retry_statuses }
|
30
|
+
assert_equal [], d.instance.instance_eval { @statuses }
|
31
|
+
assert_equal 2.0, d.instance.instance_eval { @http_read_timeout }
|
32
|
+
assert_equal 2.0, d.instance.instance_eval { @http_open_timeout }
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_invalid_endpoint
|
36
|
+
assert_raise Fluent::ConfigError do
|
37
|
+
create_driver('endpoint_url \\@3')
|
38
|
+
end
|
39
|
+
|
40
|
+
assert_raise Fluent::ConfigError do
|
41
|
+
create_driver('endpoint_url google.com')
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_write_status_retry
|
46
|
+
setup_rspec(self)
|
47
|
+
|
48
|
+
d = create_driver(%[
|
49
|
+
endpoint_url http://local.endpoint
|
50
|
+
http_retry_statuses 500
|
51
|
+
])
|
52
|
+
|
53
|
+
d.emit('abc')
|
54
|
+
|
55
|
+
http = double()
|
56
|
+
http.stub(:finish)
|
57
|
+
http.stub(:start).and_yield(http)
|
58
|
+
http.stub(:request) do
|
59
|
+
response = OpenStruct.new
|
60
|
+
response.code = '500'
|
61
|
+
response
|
62
|
+
end
|
63
|
+
|
64
|
+
d.instance.instance_eval { @http = http }
|
65
|
+
|
66
|
+
assert_raise RuntimeError do
|
67
|
+
d.run
|
68
|
+
end
|
69
|
+
|
70
|
+
verify_rspec
|
71
|
+
teardown_rspec
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_write
|
75
|
+
setup_rspec(self)
|
76
|
+
|
77
|
+
d = create_driver('endpoint_url http://www.google.com/')
|
78
|
+
|
79
|
+
d.emit('message')
|
80
|
+
http = double('Net::HTTP')
|
81
|
+
http.stub(:finish)
|
82
|
+
http.stub(:start).and_yield(http)
|
83
|
+
http.stub(:request) do |request|
|
84
|
+
assert(request.body =~ /message/)
|
85
|
+
response = OpenStruct.new
|
86
|
+
response.code = '200'
|
87
|
+
response
|
88
|
+
end
|
89
|
+
|
90
|
+
d.instance.instance_eval { @http = http }
|
91
|
+
|
92
|
+
d.run
|
93
|
+
|
94
|
+
verify_rspec
|
95
|
+
teardown_rspec
|
96
|
+
end
|
97
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
|
6
|
+
begin
|
7
|
+
Bundler.setup(:default, :development)
|
8
|
+
rescue Bundler::BundlerError => e
|
9
|
+
$stderr.puts e.message
|
10
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
11
|
+
exit e.status_code
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'test/unit'
|
15
|
+
require 'rspec/mocks'
|
16
|
+
|
17
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
18
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
19
|
+
|
20
|
+
require 'fluent/test'
|
21
|
+
require 'fluent/plugin/out_http_buffered'
|
22
|
+
|
23
|
+
# TestCase base class
|
24
|
+
class Test::Unit::TestCase
|
25
|
+
def setup_rspec(test_case)
|
26
|
+
RSpec::Mocks.setup(test_case)
|
27
|
+
end
|
28
|
+
|
29
|
+
def verify_rspec
|
30
|
+
RSpec::Mocks.verify
|
31
|
+
end
|
32
|
+
|
33
|
+
def teardown_rspec
|
34
|
+
RSpec::Mocks.teardown
|
35
|
+
end
|
36
|
+
end
|
metadata
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-out-http-buffered
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alexander Blagoev
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-05-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.10.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.10.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.9.2
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.9.2
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec-mocks
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 2.13.0
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 2.13.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.3.4
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.3.4
|
69
|
+
description: Send fluent buffered logs to an http endpoint
|
70
|
+
email: alexander.i.blagoev@gmail.com
|
71
|
+
executables: []
|
72
|
+
extensions: []
|
73
|
+
extra_rdoc_files:
|
74
|
+
- LICENSE.txt
|
75
|
+
- README.rdoc
|
76
|
+
files:
|
77
|
+
- lib/fluent/plugin/out_http_buffered.rb
|
78
|
+
- Gemfile
|
79
|
+
- LICENSE.txt
|
80
|
+
- README.rdoc
|
81
|
+
- Rakefile
|
82
|
+
- VERSION
|
83
|
+
- fluent-plugin-out-http-buffered.gemspec
|
84
|
+
- test/helper.rb
|
85
|
+
- test/fluent/plugin/test_out_http_buffered.rb
|
86
|
+
homepage: http://github.com/ablagoev/fluent-plugin-out-http-buffered
|
87
|
+
licenses:
|
88
|
+
- MIT
|
89
|
+
metadata: {}
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ! '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubyforge_project:
|
106
|
+
rubygems_version: 2.0.3
|
107
|
+
signing_key:
|
108
|
+
specification_version: 4
|
109
|
+
summary: Fluentd http buffered output plugin
|
110
|
+
test_files: []
|