syslogstash 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/LICENCE +674 -0
- data/README.md +93 -0
- data/bin/syslogstash +51 -0
- data/lib/.gitkeep +0 -0
- data/lib/syslogstash.rb +26 -0
- data/lib/syslogstash/logstash_writer.rb +110 -0
- data/lib/syslogstash/syslog_reader.rb +123 -0
- data/lib/syslogstash/worker.rb +26 -0
- data/syslogstash.gemspec +37 -0
- metadata +201 -0
data/README.md
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
Feed everything from one or more syslog pipes to a logstash server.
|
2
|
+
|
3
|
+
# Installation
|
4
|
+
|
5
|
+
It's a gem:
|
6
|
+
|
7
|
+
gem install syslogstash
|
8
|
+
|
9
|
+
There's also the wonders of [the Gemfile](http://bundler.io):
|
10
|
+
|
11
|
+
gem 'syslogstash'
|
12
|
+
|
13
|
+
If you're the sturdy type that likes to run from git:
|
14
|
+
|
15
|
+
rake install
|
16
|
+
|
17
|
+
Or, if you've eschewed the convenience of Rubygems entirely, then you
|
18
|
+
presumably know what to do already.
|
19
|
+
|
20
|
+
|
21
|
+
# Usage
|
22
|
+
|
23
|
+
Write a configuration file, then start `syslogstash` giving the name of the
|
24
|
+
config file as an argument:
|
25
|
+
|
26
|
+
syslogstash /etc/syslogstash.conf
|
27
|
+
|
28
|
+
## Config File Format
|
29
|
+
|
30
|
+
The file which describes how `syslogstash` will operate is a fairly simple
|
31
|
+
YAML file. It consists of two sections, `sockets` and `servers`, which list
|
32
|
+
the UNIX sockets to listen for syslog messages on, and the URLs of logstash
|
33
|
+
servers to send the resulting log entries to. Optionally, you can specify
|
34
|
+
additional tags to insert into every message received from each syslog
|
35
|
+
socket.
|
36
|
+
|
37
|
+
It looks like this:
|
38
|
+
|
39
|
+
sockets:
|
40
|
+
# These sockets have no additional tags
|
41
|
+
/tmp/sock1:
|
42
|
+
/tmp/sock2:
|
43
|
+
|
44
|
+
# This socket will have its messages tagged
|
45
|
+
/tmp/taggedsock:
|
46
|
+
foo: bar
|
47
|
+
baz: wombat
|
48
|
+
|
49
|
+
# Every log entry received will be sent to *exactly* one of these
|
50
|
+
# servers. This provides high availability for your log messages.
|
51
|
+
# NOTE: Only tcp:// URLs are supported.
|
52
|
+
servers:
|
53
|
+
- tcp://10.0.0.1:5151
|
54
|
+
- tcp://10.0.0.2:5151
|
55
|
+
|
56
|
+
|
57
|
+
## Logstash server configuration
|
58
|
+
|
59
|
+
You'll need to setup a TCP input, with the `json_lines` codec, for
|
60
|
+
`syslogstash` to send log entries to. It can look as simple as this:
|
61
|
+
|
62
|
+
tcp {
|
63
|
+
port => 5151
|
64
|
+
codec => "json_lines"
|
65
|
+
}
|
66
|
+
|
67
|
+
|
68
|
+
# Contributing
|
69
|
+
|
70
|
+
Bug reports should be sent to the [Github issue
|
71
|
+
tracker](https://github.com/discourse/syslogstash/issues).
|
72
|
+
Patches can be sent as a [Github pull
|
73
|
+
request](https://github.com/discourse/syslogstash/pulls].
|
74
|
+
|
75
|
+
|
76
|
+
# Licence
|
77
|
+
|
78
|
+
Unless otherwise stated, everything in this repo is covered by the following
|
79
|
+
copyright notice:
|
80
|
+
|
81
|
+
Copyright (C) 2015 Civilized Discourse Construction Kit Inc.
|
82
|
+
|
83
|
+
This program is free software: you can redistribute it and/or modify it
|
84
|
+
under the terms of the GNU General Public License version 3, as
|
85
|
+
published by the Free Software Foundation.
|
86
|
+
|
87
|
+
This program is distributed in the hope that it will be useful,
|
88
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
89
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
90
|
+
GNU General Public License for more details.
|
91
|
+
|
92
|
+
You should have received a copy of the GNU General Public License
|
93
|
+
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
data/bin/syslogstash
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'syslogstash'
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
if ARGV.length != 1
|
7
|
+
$stderr.puts <<-EOF.gsub(/^\t\t/, '')
|
8
|
+
Invalid usage
|
9
|
+
|
10
|
+
Usage:
|
11
|
+
#{$0} <configfile>
|
12
|
+
EOF
|
13
|
+
|
14
|
+
exit 1
|
15
|
+
end
|
16
|
+
|
17
|
+
unless File.exist?(ARGV[0])
|
18
|
+
$stderr.puts "Config file #{ARGV[0]} does not exist"
|
19
|
+
exit 1
|
20
|
+
end
|
21
|
+
|
22
|
+
unless File.readable?(ARGV[0])
|
23
|
+
$stderr.puts "Config file #{ARGV[0]} not readable"
|
24
|
+
exit 1
|
25
|
+
end
|
26
|
+
|
27
|
+
cfg = YAML.load_file(ARGV[0])
|
28
|
+
|
29
|
+
unless cfg.is_a? Hash
|
30
|
+
$stderr.puts "Config file #{ARGV[0]} does not contain a YAML hash"
|
31
|
+
exit 1
|
32
|
+
end
|
33
|
+
|
34
|
+
%w{sockets servers}.each do |section|
|
35
|
+
unless cfg.has_key?(section)
|
36
|
+
$stderr.puts "Config file #{ARGV[0]} does not have a '#{section}' section"
|
37
|
+
exit 1
|
38
|
+
end
|
39
|
+
|
40
|
+
unless cfg[section].respond_to?(:empty?)
|
41
|
+
$stderr.puts "Config file #{ARGV[0]} has a malformed '#{section}' section"
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
|
45
|
+
if cfg[section].empty?
|
46
|
+
$stderr.puts "Config file #{ARGV[0]} has an empty '#{section}' section"
|
47
|
+
exit 1
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
Syslogstash.new(cfg['sockets'], cfg['servers']).run
|
data/lib/.gitkeep
ADDED
File without changes
|
data/lib/syslogstash.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'socket'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
# Read syslog messages from one or more sockets, and send it to a logstash
|
6
|
+
# server.
|
7
|
+
#
|
8
|
+
class Syslogstash
|
9
|
+
def initialize(sockets, servers)
|
10
|
+
@writer = LogstashWriter.new(servers)
|
11
|
+
|
12
|
+
@readers = sockets.map { |f, tags| SyslogReader.new(f, tags, @writer) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
@writer.run
|
17
|
+
@readers.each { |w| w.run }
|
18
|
+
|
19
|
+
@writer.wait
|
20
|
+
@readers.each { |w| w.wait }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
require_relative 'syslogstash/syslog_reader'
|
25
|
+
require_relative 'syslogstash/logstash_writer'
|
26
|
+
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require_relative 'worker'
|
2
|
+
|
3
|
+
# Write messages to one of a collection of logstash servers.
|
4
|
+
#
|
5
|
+
class Syslogstash::LogstashWriter
|
6
|
+
include Syslogstash::Worker
|
7
|
+
|
8
|
+
# Create a new logstash writer.
|
9
|
+
#
|
10
|
+
# Give it a list of servers, and your writer will be ready to go.
|
11
|
+
# No messages will actually be *delivered*, though, until you call #run.
|
12
|
+
#
|
13
|
+
def initialize(servers)
|
14
|
+
@servers = servers.map { |s| URI(s) }
|
15
|
+
|
16
|
+
unless @servers.all? { |url| url.scheme == 'tcp' }
|
17
|
+
raise ArgumentError,
|
18
|
+
"Unsupported URL scheme: #{@servers.select { |url| url.scheme != 'tcp' }.join(', ')}"
|
19
|
+
end
|
20
|
+
|
21
|
+
@entries = []
|
22
|
+
@entries_mutex = Mutex.new
|
23
|
+
end
|
24
|
+
|
25
|
+
# Add an entry to the list of messages to be sent to logstash. Actual
|
26
|
+
# message delivery will happen in a worker thread that is started with
|
27
|
+
# #run.
|
28
|
+
#
|
29
|
+
def send_entry(e)
|
30
|
+
@entries_mutex.synchronize { @entries << e }
|
31
|
+
@worker.run if @worker
|
32
|
+
end
|
33
|
+
|
34
|
+
# Start sending messages to logstash servers. This method will return
|
35
|
+
# almost immediately, and actual message sending will occur in a
|
36
|
+
# separate worker thread.
|
37
|
+
#
|
38
|
+
def run
|
39
|
+
@worker = Thread.new { send_messages }
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def send_messages
|
45
|
+
loop do
|
46
|
+
if @entries_mutex.synchronize { @entries.empty? }
|
47
|
+
sleep 1
|
48
|
+
else
|
49
|
+
begin
|
50
|
+
entry = @entries_mutex.synchronize { @entries.shift }
|
51
|
+
|
52
|
+
current_server do |s|
|
53
|
+
s.puts entry
|
54
|
+
end
|
55
|
+
|
56
|
+
# If we got here, we sent successfully, so we don't want
|
57
|
+
# to put the entry back on the queue in the ensure block
|
58
|
+
entry = nil
|
59
|
+
ensure
|
60
|
+
@entries_mutex.synchronize { @entries.unshift if entry }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# *Yield* a TCPSocket connected to the server we currently believe to
|
67
|
+
# be accepting log entries, so that something can send log entries to
|
68
|
+
# it.
|
69
|
+
#
|
70
|
+
# The yielding is very deliberate: it allows us to centralise all
|
71
|
+
# error detection and handling within this one method, and retry
|
72
|
+
# sending just be calling `yield` again when we've connected to
|
73
|
+
# another server.
|
74
|
+
#
|
75
|
+
def current_server
|
76
|
+
# I could handle this more cleanly with recursion, but I don't want
|
77
|
+
# to fill the stack if we have to retry a lot of times
|
78
|
+
done = false
|
79
|
+
|
80
|
+
until done
|
81
|
+
if @current_server
|
82
|
+
begin
|
83
|
+
debug { "Using current server" }
|
84
|
+
yield @current_server
|
85
|
+
done = true
|
86
|
+
rescue SystemCallError => ex
|
87
|
+
# Something went wrong during the send; disconnect from this
|
88
|
+
# server and recycle
|
89
|
+
debug { "Error while writing to current server: #{ex.message} (#{ex.class})" }
|
90
|
+
@current_server.close
|
91
|
+
@current_server = nil
|
92
|
+
sleep 0.1
|
93
|
+
end
|
94
|
+
else
|
95
|
+
begin
|
96
|
+
# Rotate the next server onto the back of the list
|
97
|
+
next_server = @servers.shift
|
98
|
+
debug { "Trying to connect to #{next_server.to_s}" }
|
99
|
+
@servers.push(next_server)
|
100
|
+
@current_server = TCPSocket.new(next_server.host, next_server.port)
|
101
|
+
rescue SystemCallError => ex
|
102
|
+
# Connection failed for any number of reasons; try again
|
103
|
+
debug { "Failed to connect to #{next_server.to_s}: #{ex.message} (#{ex.class})" }
|
104
|
+
sleep 0.1
|
105
|
+
retry
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require_relative 'worker'
|
2
|
+
|
3
|
+
# A single socket reader.
|
4
|
+
#
|
5
|
+
class Syslogstash::SyslogReader
|
6
|
+
include Syslogstash::Worker
|
7
|
+
|
8
|
+
def initialize(file, tags, logstash)
|
9
|
+
@file, @tags, @logstash = file, tags, logstash
|
10
|
+
end
|
11
|
+
|
12
|
+
# Start reading from the socket file, parsing entries, and flinging
|
13
|
+
# them at logstash. This method will return, with the operation
|
14
|
+
# continuing in a separate thread.
|
15
|
+
#
|
16
|
+
def run
|
17
|
+
debug { "#run called" }
|
18
|
+
|
19
|
+
socket = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM, 0)
|
20
|
+
socket.bind(Socket.pack_sockaddr_un(@file))
|
21
|
+
|
22
|
+
@worker = Thread.new do
|
23
|
+
begin
|
24
|
+
loop do
|
25
|
+
msg = socket.recvmsg
|
26
|
+
debug { "Message received: #{msg.inspect}" }
|
27
|
+
process_message msg.first.chomp
|
28
|
+
end
|
29
|
+
ensure
|
30
|
+
socket.close
|
31
|
+
File.unlink(@file) rescue nil
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def process_message(msg)
|
39
|
+
if msg =~ /^<(\d+)>(\w{3} [ 0-9]{2} [0-9:]{8}) (.*)$/
|
40
|
+
flags = $1.to_i
|
41
|
+
timestamp = $2
|
42
|
+
content = $3
|
43
|
+
|
44
|
+
# Lo! the many ways that syslog messages can be formatted
|
45
|
+
hostname, program, pid, message = case content
|
46
|
+
# the gold standard: hostname, program name with optional PID
|
47
|
+
when /^([a-zA-Z0-9_-]*[^:]) (\S+?)(\[(\d+)\])?: (.*)$/
|
48
|
+
[$1, $2, $4, $5]
|
49
|
+
# hostname, no program name
|
50
|
+
when /^([a-zA-Z0-9_-]+) (\S+[^:] .*)$/
|
51
|
+
[$1, nil, nil, $2]
|
52
|
+
# program name, no hostname (yeah, you heard me, non-RFC compliant!)
|
53
|
+
when /^(\S+?)(\[(\d+)\])?: (.*)$/
|
54
|
+
[nil, $1, $3, $4]
|
55
|
+
else
|
56
|
+
# I have NFI
|
57
|
+
[nil, nil, nil, content]
|
58
|
+
end
|
59
|
+
|
60
|
+
severity = flags % 8
|
61
|
+
facility = flags / 8
|
62
|
+
|
63
|
+
log_entry = log_entry(
|
64
|
+
syslog_timestamp: timestamp,
|
65
|
+
severity: severity,
|
66
|
+
facility: facility,
|
67
|
+
hostname: hostname,
|
68
|
+
program: program,
|
69
|
+
pid: pid,
|
70
|
+
message: message,
|
71
|
+
).to_json
|
72
|
+
|
73
|
+
@logstash.send_entry(log_entry)
|
74
|
+
else
|
75
|
+
$stderr.puts "Unparseable message: #{msg}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def log_entry(h)
|
80
|
+
{}.tap do |e|
|
81
|
+
e['@version'] = '1'
|
82
|
+
e['@timestamp'] = Time.now.utc.strftime("%FT%T.%LZ")
|
83
|
+
|
84
|
+
h['facility_name'] = FACILITIES[h[:facility]]
|
85
|
+
h['severity_name'] = SEVERITIES[h[:severity]]
|
86
|
+
|
87
|
+
e.merge!(h.delete_if { |k,v| v.nil? })
|
88
|
+
|
89
|
+
e[:pid] = e[:pid].to_i if e.has_key?(:pid)
|
90
|
+
|
91
|
+
e.merge!(@tags) if @tags.is_a? Hash
|
92
|
+
|
93
|
+
debug { "Log entry is: #{e.inspect}" }
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
FACILITIES = %w{
|
98
|
+
kern
|
99
|
+
user
|
100
|
+
mail
|
101
|
+
daemon
|
102
|
+
auth
|
103
|
+
syslog
|
104
|
+
lpr
|
105
|
+
news
|
106
|
+
uucp
|
107
|
+
cron
|
108
|
+
authpriv
|
109
|
+
ftp
|
110
|
+
local0 local1 local2 local3 local4 local5 local6 local7
|
111
|
+
}
|
112
|
+
|
113
|
+
SEVERITIES = %w{
|
114
|
+
emerg
|
115
|
+
alert
|
116
|
+
crit
|
117
|
+
err
|
118
|
+
warning
|
119
|
+
notice
|
120
|
+
info
|
121
|
+
debug
|
122
|
+
}
|
123
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# Common code shared between both readers and writers.
|
2
|
+
#
|
3
|
+
module Syslogstash::Worker
|
4
|
+
# If you ever want to stop a reader, here's how.
|
5
|
+
def stop
|
6
|
+
if @worker
|
7
|
+
@worker.kill
|
8
|
+
@worker.join
|
9
|
+
@worker = nil
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
# If you want to wait for a reader to die, here's how.
|
14
|
+
#
|
15
|
+
def wait
|
16
|
+
@worker.join
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def debug
|
22
|
+
if ENV['DEBUG_SYSLOGSTASH']
|
23
|
+
puts "#{Time.now.strftime("%F %T.%L")} #{self.class} #{yield.to_s}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/syslogstash.gemspec
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
begin
|
2
|
+
require 'git-version-bump'
|
3
|
+
rescue LoadError
|
4
|
+
nil
|
5
|
+
end
|
6
|
+
|
7
|
+
Gem::Specification.new do |s|
|
8
|
+
s.name = "syslogstash"
|
9
|
+
|
10
|
+
s.version = GVB.version rescue "0.0.0.1.NOGVB"
|
11
|
+
s.date = GVB.date rescue Time.now.strftime("%Y-%m-%d")
|
12
|
+
|
13
|
+
s.platform = Gem::Platform::RUBY
|
14
|
+
|
15
|
+
s.summary = "Send messages from syslog UNIX sockets to logstash"
|
16
|
+
|
17
|
+
s.authors = ["Matt Palmer"]
|
18
|
+
s.email = ["matt.palmer@discourse.org"]
|
19
|
+
s.homepage = "https://github.com/discourse/syslogstash"
|
20
|
+
|
21
|
+
s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Rakefile)/ }
|
22
|
+
s.executables = ["syslogstash"]
|
23
|
+
|
24
|
+
s.required_ruby_version = ">= 2.1.0"
|
25
|
+
|
26
|
+
s.add_development_dependency 'bundler'
|
27
|
+
s.add_development_dependency 'github-release'
|
28
|
+
s.add_development_dependency 'guard-spork'
|
29
|
+
s.add_development_dependency 'guard-rspec'
|
30
|
+
s.add_development_dependency 'rake', '~> 10.4', '>= 10.4.2'
|
31
|
+
# Needed for guard
|
32
|
+
s.add_development_dependency 'rb-inotify', '~> 0.9'
|
33
|
+
s.add_development_dependency 'redcarpet'
|
34
|
+
s.add_development_dependency 'rspec'
|
35
|
+
s.add_development_dependency 'webmock'
|
36
|
+
s.add_development_dependency 'yard'
|
37
|
+
end
|