fluent-plugin-out-http-buffered 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ OWU0ZDllMjIzMTA1MjkxZWVlNDdiMzJlZGZiMGUzYWExNWNiZjEwMg==
5
+ data.tar.gz: !binary |-
6
+ OTExYzg3MDQ2YWZiYjQwZTVhOTI1ZGY4ZjVkOGJjMmQ1OTQ3NGU3OA==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ OGY5MDdlMGM5ZGVhZDIxMTU0ZjI4ZWIyYTJhY2Q0YTM0NjUzYTI4ZWU2OGEx
10
+ Yzg4Yzc1OGEzN2M5ODQ1NWJhMGI4NTcxMWExM2IzOTE3NDVmYTMxNDBmOGRm
11
+ NTdiY2Y1YTA5YTY1MmVjZGEwMTQ5MmE4MWQ5NjBlMjYxMjJjYWU=
12
+ data.tar.gz: !binary |-
13
+ ZjcxOThiZWYzNTMxMzUwMTUwYThjOTBlMTg1NmY0NjVlM2I4N2I3ZDU3NmMx
14
+ MjI2YzVhOTE0ZTdhN2Y4ZjA5MDkwMTM0OTZmNTgwZGI0N2IwMWY2MmEyYzM2
15
+ Y2JjMDYxNTlkYzE1MzQyYWJiMGYyYTY3OGMxNTdlNjA0YTQ3OWM=
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2013 ablagoev
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,36 @@
1
+ = fluent-out-http-buffered {<img src="https://travis-ci.org/ablagoev/fluent-plugin-out-http-buffered.png?branch=master" />}[https://travis-ci.org/ablagoev/fluent-plugin-out-http-buffered]
2
+
3
+ This is an output plugin for (Fluentd)[http://fluentd.org/] which deliveres buffered log messages to an http endpoint.
4
+
5
+ It has configurable (read)[http://ruby-doc.org/stdlib-2.0/libdoc/net/http/rdoc/Net/HTTP.html#method-i-read_timeout-3D] and (open)[http://ruby-doc.org/stdlib-2.0/libdoc/net/http/rdoc/Net/HTTP.html#open_timeout] timeouts.
6
+
7
+ Clients can also configure which http response statuses should be retried (in most cases clients would want to retry on status 500).
8
+
9
+ All messages are sent through POST in json format.
10
+
11
+ The plugin was influenced by the standard (http output plugin)[https://github.com/ento/fluent-plugin-out-http].
12
+
13
+ == Installation:
14
+
15
+ `gem install fluent-out-http-buffered`
16
+
17
+ == Usage:
18
+
19
+ # Configuration file fluent.conf
20
+ <match fluentd.test.*>
21
+ type http_buffered
22
+ flush_interval 2s
23
+ #Endpoint for messages
24
+ endpoint_url http://localhost/fluent.php
25
+ #Comma separated list of http statuses which need to be retried
26
+ http_retry_statuses 500, 403
27
+ #Read timeout in seconds, supports floats
28
+ http_read_timeout 2.2
29
+ #Open timeout in seconds, supports floats
30
+ http_open_timeout 2.34
31
+ </match>
32
+
33
+ == Copyright
34
+
35
+ Copyright (c) 2013 ablagoev. See LICENSE.txt for
36
+ further details.
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.0.1
@@ -0,0 +1,39 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-out-http-buffered'
6
+ s.version = File.read("VERSION").strip
7
+ s.date = '2013-05-13'
8
+ s.summary = "Fluentd http buffered output plugin"
9
+ s.description = "Send fluent buffered logs to an http endpoint"
10
+ s.authors = ["Alexander Blagoev"]
11
+ s.email = 'alexander.i.blagoev@gmail.com'
12
+ s.homepage =
13
+ 'http://github.com/ablagoev/fluent-plugin-out-http-buffered'
14
+
15
+ s.files = [
16
+ "lib/fluent/plugin/out_http_buffered.rb",
17
+ "Gemfile",
18
+ "LICENSE.txt",
19
+ "README.rdoc",
20
+ "Rakefile",
21
+ "VERSION",
22
+ "fluent-plugin-out-http-buffered.gemspec",
23
+ "test/helper.rb",
24
+ "test/fluent/plugin/test_out_http_buffered.rb",
25
+ ]
26
+
27
+ s.extra_rdoc_files = [
28
+ "LICENSE.txt",
29
+ "README.rdoc"
30
+ ]
31
+ s.licenses = ["MIT"]
32
+
33
+ s.require_paths = ['lib']
34
+
35
+ s.add_dependency "fluentd", "~> 0.10.0"
36
+ s.add_development_dependency "rake", ">= 0.9.2"
37
+ s.add_development_dependency "rspec-mocks", ">= 2.13.0"
38
+ s.add_development_dependency "bundler", ">= 1.3.4"
39
+ end
@@ -0,0 +1,109 @@
1
+ # encoding: utf-8
2
+
3
+ module Fluent
4
+ # Main Output plugin class
5
+ class HttpBufferedOutput < Fluent::BufferedOutput
6
+ Fluent::Plugin.register_output('http_buffered', self)
7
+
8
+ def initialize
9
+ super
10
+ require 'net/http'
11
+ require 'uri'
12
+ end
13
+
14
+ # Endpoint URL ex. localhost.local/api/
15
+ config_param :endpoint_url, :string
16
+
17
+ # statuses under which to retry
18
+ config_param :http_retry_statuses, :string, default: ''
19
+
20
+ # read timeout for the http call
21
+ config_param :http_read_timeout, :float, default: 2.0
22
+
23
+ # open timeout for the http call
24
+ config_param :http_open_timeout, :float, default: 2.0
25
+
26
+ def configure(conf)
27
+ super
28
+
29
+ # Check if endpoint URL is valid
30
+ unless @endpoint_url =~ /^#{URI.regexp}$/
31
+ fail Fluent::ConfigError, 'endpoint_url invalid'
32
+ end
33
+
34
+ begin
35
+ @uri = URI.parse(@endpoint_url)
36
+ rescue URI::InvalidURIError
37
+ raise Fluent::ConfigError, 'endpoint_url invalid'
38
+ end
39
+
40
+ # Parse http statuses
41
+ @statuses = @http_retry_statuses.split(',').map { |status| status.to_i }
42
+
43
+ @statuses = [] if @statuses.nil?
44
+
45
+ @http = Net::HTTP.new(@uri.host, @uri.port)
46
+ @http.read_timeout = @http_read_timeout
47
+ @http.open_timeout = @http_open_timeout
48
+ end
49
+
50
+ def start
51
+ super
52
+ end
53
+
54
+ def shutdown
55
+ super
56
+ begin
57
+ @http.finish
58
+ rescue
59
+ end
60
+ end
61
+
62
+ def format(tag, time, record)
63
+ [tag, time, record].to_msgpack
64
+ end
65
+
66
+ def write(chunk)
67
+ data = []
68
+ chunk.msgpack_each do |(tag, time, record)|
69
+ data << [tag, time, record]
70
+ end
71
+
72
+ request = create_request(data)
73
+
74
+ begin
75
+ response = @http.start do |http|
76
+ request = create_request(data)
77
+ http.request request
78
+ end
79
+
80
+ if @statuses.include? response.code.to_i
81
+ # Raise an exception so that fluent retries
82
+ fail "Server returned bad status: #{response.code}"
83
+ end
84
+ rescue IOError, EOFError, SystemCallError => e
85
+ # server didn't respond
86
+ $log.warn "Net::HTTP.#{request.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
87
+ ensure
88
+ begin
89
+ @http.finish
90
+ rescue
91
+ end
92
+ end
93
+ end
94
+
95
+ protected
96
+
97
+ def create_request(data)
98
+ request = Net::HTTP::Post.new(@uri.request_uri)
99
+
100
+ # Headers
101
+ request['Content-Type'] = 'application/json'
102
+
103
+ # Body
104
+ request.body = JSON.dump(data)
105
+
106
+ request
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,97 @@
1
+ # encoding: utf-8
2
+
3
+ require 'helper'
4
+ require 'yaml'
5
+
6
+ # Test case for the output plugin
7
+ class HttpBufferedOutputTest < Test::Unit::TestCase
8
+ def setup
9
+ Fluent::Test.setup
10
+ end
11
+
12
+ CONFIG = %[
13
+ endpoint_url http://local.endpoint
14
+ ]
15
+
16
+ # Used to test invalid method config
17
+ CONFIG_METHOD = %[
18
+ endpoint_url local.endpoint
19
+ http_method invalid_method
20
+ ]
21
+
22
+ def create_driver(conf = CONFIG)
23
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::HttpBufferedOutput).configure(conf)
24
+ end
25
+
26
+ def test_configure
27
+ d = create_driver
28
+ assert_equal 'http://local.endpoint', d.instance.instance_eval { @endpoint_url }
29
+ assert_equal '', d.instance.instance_eval { @http_retry_statuses }
30
+ assert_equal [], d.instance.instance_eval { @statuses }
31
+ assert_equal 2.0, d.instance.instance_eval { @http_read_timeout }
32
+ assert_equal 2.0, d.instance.instance_eval { @http_open_timeout }
33
+ end
34
+
35
+ def test_invalid_endpoint
36
+ assert_raise Fluent::ConfigError do
37
+ create_driver('endpoint_url \\@3')
38
+ end
39
+
40
+ assert_raise Fluent::ConfigError do
41
+ create_driver('endpoint_url google.com')
42
+ end
43
+ end
44
+
45
+ def test_write_status_retry
46
+ setup_rspec(self)
47
+
48
+ d = create_driver(%[
49
+ endpoint_url http://local.endpoint
50
+ http_retry_statuses 500
51
+ ])
52
+
53
+ d.emit('abc')
54
+
55
+ http = double()
56
+ http.stub(:finish)
57
+ http.stub(:start).and_yield(http)
58
+ http.stub(:request) do
59
+ response = OpenStruct.new
60
+ response.code = '500'
61
+ response
62
+ end
63
+
64
+ d.instance.instance_eval { @http = http }
65
+
66
+ assert_raise RuntimeError do
67
+ d.run
68
+ end
69
+
70
+ verify_rspec
71
+ teardown_rspec
72
+ end
73
+
74
+ def test_write
75
+ setup_rspec(self)
76
+
77
+ d = create_driver('endpoint_url http://www.google.com/')
78
+
79
+ d.emit('message')
80
+ http = double('Net::HTTP')
81
+ http.stub(:finish)
82
+ http.stub(:start).and_yield(http)
83
+ http.stub(:request) do |request|
84
+ assert(request.body =~ /message/)
85
+ response = OpenStruct.new
86
+ response.code = '200'
87
+ response
88
+ end
89
+
90
+ d.instance.instance_eval { @http = http }
91
+
92
+ d.run
93
+
94
+ verify_rspec
95
+ teardown_rspec
96
+ end
97
+ end
@@ -0,0 +1,36 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+
6
+ begin
7
+ Bundler.setup(:default, :development)
8
+ rescue Bundler::BundlerError => e
9
+ $stderr.puts e.message
10
+ $stderr.puts 'Run `bundle install` to install missing gems'
11
+ exit e.status_code
12
+ end
13
+
14
+ require 'test/unit'
15
+ require 'rspec/mocks'
16
+
17
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
18
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
19
+
20
+ require 'fluent/test'
21
+ require 'fluent/plugin/out_http_buffered'
22
+
23
+ # TestCase base class
24
+ class Test::Unit::TestCase
25
+ def setup_rspec(test_case)
26
+ RSpec::Mocks.setup(test_case)
27
+ end
28
+
29
+ def verify_rspec
30
+ RSpec::Mocks.verify
31
+ end
32
+
33
+ def teardown_rspec
34
+ RSpec::Mocks.teardown
35
+ end
36
+ end
metadata ADDED
@@ -0,0 +1,110 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-out-http-buffered
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Alexander Blagoev
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-05-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: 0.10.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 0.10.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ! '>='
32
+ - !ruby/object:Gem::Version
33
+ version: 0.9.2
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ! '>='
39
+ - !ruby/object:Gem::Version
40
+ version: 0.9.2
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec-mocks
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ! '>='
46
+ - !ruby/object:Gem::Version
47
+ version: 2.13.0
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: 2.13.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 1.3.4
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ! '>='
67
+ - !ruby/object:Gem::Version
68
+ version: 1.3.4
69
+ description: Send fluent buffered logs to an http endpoint
70
+ email: alexander.i.blagoev@gmail.com
71
+ executables: []
72
+ extensions: []
73
+ extra_rdoc_files:
74
+ - LICENSE.txt
75
+ - README.rdoc
76
+ files:
77
+ - lib/fluent/plugin/out_http_buffered.rb
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.rdoc
81
+ - Rakefile
82
+ - VERSION
83
+ - fluent-plugin-out-http-buffered.gemspec
84
+ - test/helper.rb
85
+ - test/fluent/plugin/test_out_http_buffered.rb
86
+ homepage: http://github.com/ablagoev/fluent-plugin-out-http-buffered
87
+ licenses:
88
+ - MIT
89
+ metadata: {}
90
+ post_install_message:
91
+ rdoc_options: []
92
+ require_paths:
93
+ - lib
94
+ required_ruby_version: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - ! '>='
97
+ - !ruby/object:Gem::Version
98
+ version: '0'
99
+ required_rubygems_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ! '>='
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ requirements: []
105
+ rubyforge_project:
106
+ rubygems_version: 2.0.3
107
+ signing_key:
108
+ specification_version: 4
109
+ summary: Fluentd http buffered output plugin
110
+ test_files: []