fluent-plugin-remote-syslog 1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: c8779785fa2d22e09a93e60c4d1aa5d47c758155
4
+ data.tar.gz: 14c7ca85158b96ef3e322acf589c627e0b3c73ee
5
+ SHA512:
6
+ metadata.gz: 875236530f7c6c97b34959bd5c1c56b97643d30696c58cd377d0632619cbe9302a3fcbb992de1c2d829d9dd8f41b24fb26fbebd766aae91ca52e5a9464c4997d
7
+ data.tar.gz: a52bd2123e0ccb0ed7f8863e0835270a0772a89328c03034aaf2d1cd13f78d86dd93b8a864e5d2446c6398360d70882c9582b09d4a75e9e4400b999c9bc9a69b
@@ -0,0 +1 @@
1
+ .idea
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
@@ -0,0 +1,39 @@
1
+ fluent-plugin-remote-syslog
2
+ ===========================
3
+
4
+ fluentd plugin for streaming logs out to a remote syslog server or syslog SaaS service (like Papertrail)
5
+
6
+ #Available Plugins:
7
+ * out_syslog: registers itself as "syslog", and is the non-buffered implementation, communicating through UDP
8
+ * out_syslog_buffered: registers itself as "syslog_buffered", and is the buffered implementation, communicating through TCP
9
+
10
+ #Plugin Settings:
11
+ Both plugins have the same configuration options:
12
+
13
+ * remote_syslog: fqdn or ip of the remote syslog instance
14
+ * port: the port, where the remote syslog instance is listening
15
+ * hostname: hostname to be set for syslog messages
16
+ * remove_tag_prefix: remove tag prefix for tag placeholder.
17
+ * tag_key: use the field specified in tag_key from record to set the syslog key
18
+ * facility: Syslog log facility
19
+ * severity: Syslog log severity
20
+ * use_record: Use severity and facility from record if available
21
+ * payload_key: Use the field specified in payload_key from record to set payload
22
+
23
+ #Configuration example:
24
+ ```
25
+ <match site.*>
26
+ type syslog_buffered
27
+ remote_syslog your.syslog.host
28
+ port 25
29
+ hostname ${hostname}
30
+ facility local0
31
+ severity debug
32
+ </match>
33
+ ```
34
+
35
+
36
+ Contributors:
37
+
38
+ * Andrea Spoldi
39
+ * [deathowl](http://github.com/deathowl)
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,23 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-remote-syslog"
6
+ gem.description = "Output plugin for streaming logs out to a remote syslog"
7
+ gem.homepage = "https://github.com/docebo/fluent-plugin-remote-syslog"
8
+ gem.summary = gem.description
9
+ gem.version = "1.0"
10
+ gem.authors = ["Andrea Spoldi"]
11
+ gem.email = "devops@docebo.com"
12
+ gem.has_rdoc = false
13
+ gem.license = 'MIT'
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_dependency "fluentd", "~> 0.10.45"
20
+ gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
21
+ gem.add_dependency "syslog_protocol"
22
+ gem.add_development_dependency "rake", ">= 0.9.2"
23
+ end
@@ -0,0 +1,103 @@
1
+ require 'fluent/mixin/config_placeholders'
2
+ module Fluent
3
+
4
+ class SyslogOutput < Fluent::Output
5
+ # First, register the plugin. NAME is the name of this plugin
6
+ # and identifies the plugin in the configuration file.
7
+ Fluent::Plugin.register_output('syslog', self)
8
+
9
+ # This method is called before starting.
10
+
11
+ config_param :remote_syslog, :string, :default => nil
12
+ config_param :port, :integer, :default => 25
13
+ config_param :hostname, :string, :default => ""
14
+ config_param :remove_tag_prefix, :string, :default => nil
15
+ config_param :tag_key, :string, :default => nil
16
+ config_param :facility, :string, :default => 'user'
17
+ config_param :severity, :string, :default => 'debug'
18
+ config_param :use_record, :string, :default => nil
19
+ config_param :payload_key, :string, :default => 'message'
20
+
21
+
22
+ def initialize
23
+ super
24
+ require 'socket'
25
+ require 'syslog_protocol'
26
+ end
27
+
28
+ def configure(conf)
29
+ super
30
+ if not conf['remote_syslog']
31
+ raise Fluent::ConfigError.new("remote syslog required")
32
+ end
33
+ @socket = UDPSocket.new
34
+ @packet = SyslogProtocol::Packet.new
35
+ if remove_tag_prefix = conf['remove_tag_prefix']
36
+ @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
37
+ end
38
+ @facilty = conf['facility']
39
+ @severity = conf['severity']
40
+ @use_record = conf['use_record']
41
+ @payload_key = conf['payload_key']
42
+ if not @payload_key
43
+ @payload_key = "message"
44
+ end
45
+ end
46
+
47
+
48
+ # This method is called when starting.
49
+ def start
50
+ super
51
+ end
52
+
53
+ # This method is called when shutting down.
54
+ def shutdown
55
+ super
56
+ end
57
+
58
+ # This method is called when an event reaches Fluentd.
59
+ # 'es' is a Fluent::EventStream object that includes multiple events.
60
+ # You can use 'es.each {|time,record| ... }' to retrieve events.
61
+ # 'chain' is an object that manages transactions. Call 'chain.next' at
62
+ # appropriate points and rollback if it raises an exception.
63
+ def emit(tag, es, chain)
64
+ tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
65
+ chain.next
66
+ es.each {|time,record|
67
+ @packet.hostname = hostname
68
+ if @use_record
69
+ @packet.facility = record['facility'] || @facilty
70
+ @packet.severity = record['severity'] || @severity
71
+ else
72
+ @packet.facility = @facilty
73
+ @packet.severity = @severity
74
+ end
75
+ if record['time']
76
+ time = Time.parse(record['time'])
77
+ else
78
+ time = Time.now
79
+ end
80
+ @packet.time = time
81
+ @packet.tag = if tag_key
82
+ record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
83
+ else
84
+ tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
85
+ end
86
+ packet = @packet.dup
87
+ packet.content = record[@payload_key]
88
+ @socket.send(packet.assemble, 0, @remote_syslog, @port)
89
+ }
90
+ end
91
+ end
92
+ class Time < Time
93
+ def timezone(timezone = 'UTC')
94
+ old = ENV['TZ']
95
+ utc = self.dup.utc
96
+ ENV['TZ'] = timezone
97
+ output = utc.localtime
98
+ ENV['TZ'] = old
99
+ output
100
+ end
101
+ end
102
+
103
+ end
@@ -0,0 +1,146 @@
1
+ require 'fluent/mixin/config_placeholders'
2
+ module Fluent
3
+ class SyslogBufferedOutput < Fluent::BufferedOutput
4
+ # First, register the plugin. NAME is the name of this plugin
5
+ # and identifies the plugin in the configuration file.
6
+ Fluent::Plugin.register_output('syslog_buffered', self)
7
+
8
+ # This method is called before starting.
9
+
10
+ config_param :remote_syslog, :string, :default => ""
11
+ config_param :port, :integer, :default => 25
12
+ config_param :hostname, :string, :default => ""
13
+ config_param :remove_tag_prefix, :string, :default => nil
14
+ config_param :tag_key, :string, :default => nil
15
+ config_param :facility, :string, :default => 'user'
16
+ config_param :severity, :string, :default => 'debug'
17
+ config_param :use_record, :string, :default => nil
18
+ config_param :payload_key, :string, :default => 'message'
19
+
20
+
21
+ def initialize
22
+ super
23
+ require 'socket'
24
+ require 'syslog_protocol'
25
+ require 'timeout'
26
+ end
27
+
28
+ def configure(conf)
29
+ super
30
+ if not conf['remote_syslog']
31
+ raise Fluent::ConfigError.new("remote syslog required")
32
+ end
33
+ @socket = create_tcp_socket(conf['remote_syslog'], conf['port'])
34
+ @packet = SyslogProtocol::Packet.new
35
+ if remove_tag_prefix = conf['remove_tag_prefix']
36
+ @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
37
+ end
38
+ @facilty = conf['facility']
39
+ @severity = conf['severity']
40
+ @use_record = conf['use_record']
41
+ @payload_key = conf['payload_key']
42
+ if not @payload_key
43
+ @payload_key = "message"
44
+ end
45
+ end
46
+
47
+ def format(tag, time, record)
48
+ [tag, time, record].to_msgpack
49
+ end
50
+
51
+ def create_tcp_socket(host, port)
52
+ begin
53
+ Timeout.timeout(10) do
54
+ begin
55
+ socket = TCPSocket.new(host, port)
56
+ rescue Errno::ENETUNREACH
57
+ retry
58
+ end
59
+ end
60
+ socket = TCPSocket.new(host, port)
61
+ secs = Integer(1)
62
+ usecs = Integer((1 - secs) * 1_000_000)
63
+ optval = [secs, usecs].pack("l_2")
64
+ socket.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
65
+ rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError, Timeout::Error => e
66
+ log.warn "out:syslog: failed to open tcp socket #{@remote_syslog}:#{@port} :#{e}"
67
+ socket = nil
68
+ end
69
+ socket
70
+ end
71
+
72
+ # This method is called when starting.
73
+ def start
74
+ super
75
+ end
76
+
77
+ # This method is called when shutting down.
78
+ def shutdown
79
+ super
80
+ end
81
+
82
+
83
+ def write(chunk)
84
+ chunk.msgpack_each {|(tag,time,record)|
85
+ send_to_syslog(tag, time, record)
86
+ }
87
+ end
88
+
89
+ def send_to_syslog(tag, time, record)
90
+ tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
91
+ @packet.hostname = hostname
92
+ if @use_record
93
+ @packet.facility = record['facility'] || @facilty
94
+ @packet.severity = record['severity'] || @severity
95
+ else
96
+ @packet.facility = @facilty
97
+ @packet.severity = @severity
98
+ end
99
+ if record['time']
100
+ time = Time.parse(record['time'])
101
+ else
102
+ time = Time.now
103
+ end
104
+ @packet.time = time
105
+ @packet.tag = if tag_key
106
+ record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
107
+ else
108
+ tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
109
+ end
110
+ packet = @packet.dup
111
+ packet.content = record[@payload_key]
112
+ begin
113
+ if not @socket
114
+ @socket = create_tcp_socket(@remote_syslog, @port)
115
+ end
116
+ if @socket
117
+ begin
118
+ @socket.write packet.assemble + "\n"
119
+ @socket.flush
120
+ rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError => e
121
+ log.warn "out:syslog: connection error by #{@remote_syslog}:#{@port} :#{e}"
122
+ @socket = nil
123
+ raise #{e}
124
+ end
125
+ else
126
+ log.warn "out:syslog: Socket connection couldn't be reestablished"
127
+ raise #{e}
128
+ end
129
+ end
130
+ end
131
+
132
+
133
+ end
134
+
135
+ class Time
136
+ def timezone(timezone = 'UTC')
137
+ old = ENV['TZ']
138
+ utc = self.dup.utc
139
+ ENV['TZ'] = timezone
140
+ output = utc.localtime
141
+ ENV['TZ'] = old
142
+ output
143
+ end
144
+ end
145
+ end
146
+
@@ -0,0 +1,30 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluentd/plugin/out_syslog'
26
+ require 'fluentd/plugin/out_syslog_buffered'
27
+
28
+
29
+ class Test::Unit::TestCase
30
+ end
@@ -0,0 +1,138 @@
1
+ require 'helper'
2
+
3
+ class SyslogOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ remote_syslog 127.0.0.1
10
+ port 25
11
+ hostname testhost
12
+ remove_tag_prefix test
13
+ severity debug
14
+ facility user
15
+ payload_key message
16
+ ]
17
+
18
+ def create_driver(conf=CONFIG,tag='test')
19
+ Fluent::Test::OutputTestDriver.new(Fluent::SyslogOutput, tag).configure(conf)
20
+ end
21
+
22
+ def test_configure
23
+ assert_raise(Fluent::ConfigError) {
24
+ d = create_driver('')
25
+ }
26
+ assert_raise(Fluent::ConfigError) {
27
+ d = create_driver %[
28
+ hostname testhost
29
+ remove_tag_prefix test
30
+ ]
31
+ }
32
+ assert_nothing_raised {
33
+ d = create_driver %[
34
+ remote_syslog 127.0.0.1
35
+ ]
36
+ }
37
+ assert_nothing_raised {
38
+ d = create_driver %[
39
+ remote_syslog 127.0.0.1
40
+ port 639
41
+ ]
42
+ }
43
+ assert_nothing_raised {
44
+ d = create_driver %[
45
+ remote_syslog 127.0.0.1
46
+ port 25
47
+ hostname deathstar
48
+ ]
49
+ }
50
+ assert_nothing_raised {
51
+ d = create_driver %[
52
+ remote_syslog 127.0.0.1
53
+ port 25
54
+ hostname testhost
55
+ remove_tag_prefix test123
56
+ ]
57
+ }
58
+ assert_nothing_raised {
59
+ d = create_driver %[
60
+ remote_syslog 127.0.0.1
61
+ port 25
62
+ hostname testhost
63
+ remove_tag_prefix test
64
+ tag_key tagtag
65
+ severity debug
66
+ ]
67
+ }
68
+ assert_nothing_raised {
69
+ d = create_driver %[
70
+ remote_syslog 127.0.0.1
71
+ port 25
72
+ hostname testhost
73
+ remove_tag_prefix test
74
+ tag_key tagtag
75
+ severity debug
76
+ facility user
77
+ ]
78
+ }
79
+ assert_nothing_raised {
80
+ d = create_driver %[
81
+ remote_syslog 127.0.0.1
82
+ port 25
83
+ hostname testhost
84
+ remove_tag_prefix test
85
+ tag_key tagtag
86
+ severity debug
87
+ facility user
88
+ payload_key message
89
+ ]
90
+ }
91
+ d = create_driver %[
92
+ remote_syslog 127.0.0.1
93
+ port 25
94
+ hostname testhost
95
+ remove_tag_prefix test
96
+ tag_key tagtag
97
+ severity debug
98
+ facility user
99
+ payload_key message
100
+ ]
101
+ assert_equal 25, d.instance.port
102
+ assert_equal "127.0.0.1", d.instance.remote_syslog
103
+ assert_equal "testhost", d.instance.hostname
104
+ assert_equal Regexp.new('^' + Regexp.escape("test")), d.instance.remove_tag_prefix
105
+ assert_equal "tagtag", d.instance.tag_key
106
+ assert_equal "debug", d.instance.severity
107
+ assert_equal "user", d.instance.facility
108
+ assert_equal "message", d.instance.payload_key
109
+
110
+ end
111
+ def test_emit
112
+ d1 = create_driver(CONFIG, 'test.in')
113
+ d1.run do
114
+ d1.emit({'message' => 'asd asd'})
115
+ d1.emit({'message' => 'dsa xasd'})
116
+ d1.emit({'message' => 'ddd ddddd'})
117
+ d1.emit({'message' => '7sssss8 ssssdasd'})
118
+ d1.emit({'message' => 'aaassddffg asdasdasfasf'})
119
+ end
120
+ assert_equal 0, d1.emits.size
121
+
122
+ end
123
+
124
+ def test_emit_with_time_and_without_time
125
+ d1 = create_driver(CONFIG, 'test.in')
126
+ d1.run do
127
+ d1.emit({'message' => 'asd asd', 'time' => '2007-01-31 12:22:26'})
128
+ d1.emit({'message' => 'dsa xasd'})
129
+ d1.emit({'message' => 'ddd ddddd', 'time' => '2007-03-01 12:22:26'})
130
+ d1.emit({'message' => '7sssss8 ssssdasd', 'time' => '2011-03-01 12:22:26'})
131
+ d1.emit({'message' => 'aaassddffg asdasdasfasf', 'time' => '2016-03-01 12:22:26'})
132
+ end
133
+ assert_equal 0, d1.emits.size
134
+
135
+ end
136
+
137
+
138
+ end
metadata ADDED
@@ -0,0 +1,110 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-remote-syslog
3
+ version: !ruby/object:Gem::Version
4
+ version: '1.0'
5
+ platform: ruby
6
+ authors:
7
+ - Andrea Spoldi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-07-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.10.45
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.10.45
27
+ - !ruby/object:Gem::Dependency
28
+ name: fluent-mixin-config-placeholders
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 0.2.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.2.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: syslog_protocol
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: 0.9.2
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: 0.9.2
69
+ description: Output plugin for streaming logs out to a remote syslog
70
+ email: devops@docebo.com
71
+ executables: []
72
+ extensions: []
73
+ extra_rdoc_files: []
74
+ files:
75
+ - ".gitignore"
76
+ - Gemfile
77
+ - README.md
78
+ - Rakefile
79
+ - fluent-plugin-remote-syslog.gemspec
80
+ - lib/fluentd/plugin/out_syslog.rb
81
+ - lib/fluentd/plugin/out_syslog_buffered.rb
82
+ - test/helper.rb
83
+ - test/plugin/test_out_syslog.rb
84
+ homepage: https://github.com/docebo/fluent-plugin-remote-syslog
85
+ licenses:
86
+ - MIT
87
+ metadata: {}
88
+ post_install_message:
89
+ rdoc_options: []
90
+ require_paths:
91
+ - lib
92
+ required_ruby_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ required_rubygems_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ requirements: []
103
+ rubyforge_project:
104
+ rubygems_version: 2.5.1
105
+ signing_key:
106
+ specification_version: 4
107
+ summary: Output plugin for streaming logs out to a remote syslog
108
+ test_files:
109
+ - test/helper.rb
110
+ - test/plugin/test_out_syslog.rb