logporter 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/logporter/event.rb +24 -0
- data/lib/logporter/namespace.rb +4 -0
- data/lib/logporter/protocol/syslog3164.rb +46 -0
- data/lib/logporter/server/connection.rb +113 -0
- data/lib/logporter/server/defaulthandler.rb +9 -0
- data/lib/logporter/server.rb +109 -0
- data/test/syslog.rb +87 -0
- metadata +74 -0
@@ -0,0 +1,24 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
|
3
|
+
class LogPorter::Event
|
4
|
+
attr_accessor :pri
|
5
|
+
attr_accessor :timestamp
|
6
|
+
attr_accessor :hostname
|
7
|
+
attr_accessor :message
|
8
|
+
attr_accessor :raw
|
9
|
+
|
10
|
+
# TODO(sissel): Should we include other source information like client
|
11
|
+
# address and port?
|
12
|
+
|
13
|
+
def time_iso8601
|
14
|
+
return timestamp.strftime("%Y-%m-%dT%H:%M:%S.") + timestamp.tv_usec.to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
if @raw == true
|
19
|
+
return message
|
20
|
+
else
|
21
|
+
return "<#{pri}>#{time_iso8601} #{hostname} #{message}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
require "time" # for Time.strptime
|
3
|
+
|
4
|
+
module LogPorter::Protocol::Syslog3164
|
5
|
+
def syslog3164_init
|
6
|
+
pri = "(?:<(?<pri>[0-9]{1,3})>)?"
|
7
|
+
month = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
8
|
+
day = "(?: [1-9]|[12][0-9]|3[01])"
|
9
|
+
hour = "(?:[01][0-9]|2[0-4])"
|
10
|
+
minute = "(?:[0-5][0-9])"
|
11
|
+
second = "(?:[0-5][0-9])"
|
12
|
+
|
13
|
+
time = [hour, minute, second].join(":")
|
14
|
+
|
15
|
+
timestamp = "(?<timestamp>#{month} #{day} #{time})"
|
16
|
+
hostname = "(?<hostname>[A-Za-z0-9_.:]+)"
|
17
|
+
header = timestamp + " " + hostname
|
18
|
+
message = "(?<message>[ -~]+)" # ascii 32 to 126
|
19
|
+
re = "^#{pri}#{header} #{message}$"
|
20
|
+
|
21
|
+
if RUBY_VERSION =~ /^1\.8/
|
22
|
+
# Ruby 1.8 doesn't support named captures
|
23
|
+
# replace (?<foo> with (
|
24
|
+
re = re.gsub(/\(\?<[^>]+>/, "(")
|
25
|
+
end
|
26
|
+
|
27
|
+
@syslog3164_re = Regexp.new(re)
|
28
|
+
end
|
29
|
+
|
30
|
+
def parse_rfc3164(line, event)
|
31
|
+
syslog3164_init if !@syslog3164_re
|
32
|
+
m = @syslog3164_re.match(line)
|
33
|
+
if m
|
34
|
+
# RFC3164 section 4.3.3 No PRI or Unidentifiable PRI
|
35
|
+
event.pri = m[1] || "13"
|
36
|
+
|
37
|
+
# TODO(sissel): DateTime is a very slow library, consider alternatives?
|
38
|
+
event.timestamp = Time.strptime(m[2], "%b %d %H:%M:%S")
|
39
|
+
event.hostname = m[3]
|
40
|
+
event.message = m[4]
|
41
|
+
return true
|
42
|
+
end
|
43
|
+
return false
|
44
|
+
end # def parse_rfc3164
|
45
|
+
end # module Log::Protocol::Syslog3164
|
46
|
+
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require "eventmachine"
|
2
|
+
require "logporter/event"
|
3
|
+
require "logporter/namespace"
|
4
|
+
require "logporter/protocol/syslog3164"
|
5
|
+
require "logporter/server"
|
6
|
+
require "socket"
|
7
|
+
|
8
|
+
class LogPorter::Server::Connection < EventMachine::Connection
|
9
|
+
include LogPorter::Protocol::Syslog3164
|
10
|
+
|
11
|
+
def initialize(server)
|
12
|
+
@server = server
|
13
|
+
end
|
14
|
+
|
15
|
+
def post_init
|
16
|
+
super
|
17
|
+
|
18
|
+
if @server.network == :tls
|
19
|
+
if RUBY_PLATFORM == "java"
|
20
|
+
$STDERR.puts "Warning... EventMachine doesn't support TLS on JRuby :("
|
21
|
+
end
|
22
|
+
|
23
|
+
# Calls EventMachine::Connection#start_tls
|
24
|
+
start_tls(
|
25
|
+
:private_key_file => @server.tls.private_key_file,
|
26
|
+
:cert_chain_file => @server.tls.cert_chain_file,
|
27
|
+
:verify_peer => @server.tls.verify_peer
|
28
|
+
)
|
29
|
+
end
|
30
|
+
|
31
|
+
@count = 0
|
32
|
+
|
33
|
+
case @server.wire
|
34
|
+
when :raw
|
35
|
+
class << self
|
36
|
+
alias_method :receive_line, :receive_line_raw
|
37
|
+
end
|
38
|
+
when :syslog
|
39
|
+
class << self
|
40
|
+
alias_method :receive_line, :receive_line_syslog
|
41
|
+
end
|
42
|
+
else
|
43
|
+
raise "Unsupported protocol #{@server.protocol}"
|
44
|
+
end
|
45
|
+
|
46
|
+
begin
|
47
|
+
@client_port, @client_address = Socket.unpack_sockaddr_in(get_peername)
|
48
|
+
puts "New client: #{@client_address}:#{@client_port}"
|
49
|
+
rescue => e
|
50
|
+
p e
|
51
|
+
end
|
52
|
+
end # def post_init
|
53
|
+
|
54
|
+
#def ssl_handshake_completed
|
55
|
+
# TODO(sissel): validate other pieces of the cert?
|
56
|
+
#puts get_peer_cert
|
57
|
+
#end
|
58
|
+
|
59
|
+
def receive_data(data)
|
60
|
+
if @server.network == :udp
|
61
|
+
client_port, client_address = Socket.unpack_sockaddr_in(get_peername)
|
62
|
+
else
|
63
|
+
client_port = @client_port
|
64
|
+
client_address = @client_address
|
65
|
+
end
|
66
|
+
|
67
|
+
@buffer ||= BufferedTokenizer.new
|
68
|
+
@buffer.extract(data).each do |line|
|
69
|
+
receive_line(line.chomp, client_address, client_port)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def receive_line_raw(line, address, port)
|
74
|
+
event = LogPorter::Event.new
|
75
|
+
|
76
|
+
# TODO(sissel): Look for an alternative to Time#strftime since it is
|
77
|
+
# insanely slow.
|
78
|
+
event.pri = "13" # RFC3164 says unknown pri == 13.
|
79
|
+
event.timestamp = Time.now
|
80
|
+
event.hostname = address
|
81
|
+
event.message = line
|
82
|
+
event.raw = true
|
83
|
+
|
84
|
+
@server.receive_event(event, address, port)
|
85
|
+
stats
|
86
|
+
end
|
87
|
+
|
88
|
+
def receive_line_syslog(line, address, port)
|
89
|
+
event = LogPorter::Event.new
|
90
|
+
if parse_rfc3164(line, event)
|
91
|
+
#elsif parse_rfc5424(line, event)
|
92
|
+
else
|
93
|
+
# Unknown message format, add syslog headers.
|
94
|
+
event.pri = "13" # RFC3164 says unknown pri == 13.
|
95
|
+
event.timestamp = Time.now
|
96
|
+
event.hostname = address
|
97
|
+
event.message = line
|
98
|
+
end
|
99
|
+
|
100
|
+
@server.receive_event(event, address, port)
|
101
|
+
stats
|
102
|
+
end # def receive_line_syslog
|
103
|
+
|
104
|
+
def stats
|
105
|
+
@start ||= Time.now
|
106
|
+
@count += 1
|
107
|
+
if @count % 50000 == 0
|
108
|
+
puts "Rate: #{@count / (Time.now - @start)}"
|
109
|
+
@start = Time.now
|
110
|
+
@count = 0
|
111
|
+
end
|
112
|
+
end # def stats
|
113
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
require "logporter/namespace"
|
4
|
+
|
5
|
+
class LogPorter::Server::DefaultHandler
|
6
|
+
def receive_event(event, server, client_addr, client_port)
|
7
|
+
puts "#{client_addr}:#{client_port}(#{server.network}/#{server.wire}) => #{event}"
|
8
|
+
end # def receive_event
|
9
|
+
end # class Log::Server::DefaultHandler
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
require "logporter/server/connection"
|
3
|
+
require "logporter/server/defaulthandler"
|
4
|
+
|
5
|
+
class LogPorter::Server
|
6
|
+
|
7
|
+
# Create a new class called 'TLSConfig' which simply acts as a data structure.
|
8
|
+
TLSConfig = Struct.new :private_key_file, :cert_chain_file, :verify_peer
|
9
|
+
|
10
|
+
# The port we are listening on
|
11
|
+
attr_reader :port
|
12
|
+
|
13
|
+
# TLS options, only meaningful if @network == :tls
|
14
|
+
attr_reader :tls
|
15
|
+
|
16
|
+
# The network layer, :tcp, :udp, or :tls
|
17
|
+
attr_reader :network
|
18
|
+
|
19
|
+
# The wire format (syslog, raw, etc)
|
20
|
+
attr_reader :wire
|
21
|
+
|
22
|
+
# Arbitrary attributes for this server. You can store whatever you want here.
|
23
|
+
# This is a hash
|
24
|
+
attr_reader :attributes
|
25
|
+
|
26
|
+
# Create a new server to listen with
|
27
|
+
# 'options' is a hash of:
|
28
|
+
#
|
29
|
+
# :net => the network layer to use (:udp, :tcp, :tls)
|
30
|
+
# :port => the port to listen on
|
31
|
+
# :wire => the wire format (:raw, :syslog)
|
32
|
+
# :handler => the handler instance. Must respond to 'receive_event'
|
33
|
+
def initialize(options)
|
34
|
+
@network = options[:net]
|
35
|
+
@port = options.delete(:port) || 514
|
36
|
+
@wire = options.delete(:wire) || :raw
|
37
|
+
@handler = options.delete(:handler) || LogPorter::Server::DefaultHandler.new
|
38
|
+
@attributes = options.delete(:attributes) || Hash.new
|
39
|
+
|
40
|
+
if @network == :tls
|
41
|
+
@tls = TLSConfig.new
|
42
|
+
@tls.private_key_file = options[:private_key]
|
43
|
+
@tls.cert_chain_file = options[:certificate_chain]
|
44
|
+
@tls.verify_peer = options[:verify_peer] || false
|
45
|
+
else
|
46
|
+
@tls = nil
|
47
|
+
end
|
48
|
+
end # def initialize
|
49
|
+
|
50
|
+
# start the server
|
51
|
+
public
|
52
|
+
def start
|
53
|
+
# We use next_tick here in case you are invoking this method from outside
|
54
|
+
# of EventMachine; this allows you to do this:
|
55
|
+
#
|
56
|
+
# s = LogPorter::server.new ...
|
57
|
+
# s.start
|
58
|
+
#
|
59
|
+
# EventMachine.run()
|
60
|
+
EventMachine.next_tick do
|
61
|
+
puts "Starting #{@network}/#{@port}"
|
62
|
+
begin
|
63
|
+
case @network
|
64
|
+
when :udp; start_udp_server
|
65
|
+
when :tcp; start_tcp_server
|
66
|
+
when :tls; start_tcp_server # tls is handled by tcp.
|
67
|
+
else
|
68
|
+
raise "Unknown network '#{@network}' expected :udp, :tcp, or :tls"
|
69
|
+
end
|
70
|
+
rescue => e
|
71
|
+
if @handler.respond_to?(:receive_exception)
|
72
|
+
@handler.receive_exception(e)
|
73
|
+
else
|
74
|
+
raise e
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end # def start
|
79
|
+
|
80
|
+
private
|
81
|
+
def start_udp_server
|
82
|
+
@socket = EventMachine::open_datagram_socket "0.0.0.0", @port,
|
83
|
+
LogPorter::Server::Connection, self
|
84
|
+
end # def start_udp
|
85
|
+
|
86
|
+
private
|
87
|
+
def start_tcp_server
|
88
|
+
@socket = EventMachine::start_server "0.0.0.0", @port,
|
89
|
+
LogPorter::Server::Connection, self
|
90
|
+
end # def start_tcp
|
91
|
+
|
92
|
+
# This method is invoked by LogPorter::Server::Connection
|
93
|
+
public
|
94
|
+
def receive_event(event, client_addr, client_port)
|
95
|
+
@handler.receive_event(event, self, client_addr, client_port)
|
96
|
+
end
|
97
|
+
|
98
|
+
public
|
99
|
+
def stop
|
100
|
+
case @network
|
101
|
+
when :tcp
|
102
|
+
EventMachine::stop_server(@socket)
|
103
|
+
when :tls
|
104
|
+
EventMachine::stop_server(@socket)
|
105
|
+
when :udp
|
106
|
+
@socket.close_connection(true)
|
107
|
+
end
|
108
|
+
end # def stop
|
109
|
+
end # class LogPorter::Server
|
data/test/syslog.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
|
2
|
+
require "test/unit"
|
3
|
+
require "logporter/protocol/syslog3164"
|
4
|
+
require "logporter/event"
|
5
|
+
|
6
|
+
class TestSyslog3164Parser < Test::Unit::TestCase
|
7
|
+
include LogPorter::Protocol::Syslog3164
|
8
|
+
|
9
|
+
def test_full_valid_message
|
10
|
+
messages = {
|
11
|
+
"<12>Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
|
12
|
+
:pri => "12",
|
13
|
+
:timestamp => "Mar 1 15:43:35",
|
14
|
+
:hostname => "snack",
|
15
|
+
:message => "kernel: Kernel logging (proc) stopped."
|
16
|
+
},
|
17
|
+
"<1>Jun 17 03:29:44 1.2.3.4 fancypants" => {
|
18
|
+
:pri => "1",
|
19
|
+
:timestamp => "Jun 17 03:29:44",
|
20
|
+
:hostname => "1.2.3.4",
|
21
|
+
:message => "fancypants"
|
22
|
+
},
|
23
|
+
"<100>Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
|
24
|
+
:pri => "100",
|
25
|
+
:timestamp => "Dec 31 22:00:00",
|
26
|
+
:hostname => "ffe0::1",
|
27
|
+
:message => "something[12345]: hello world",
|
28
|
+
},
|
29
|
+
}
|
30
|
+
|
31
|
+
event = LogPorter::Event.new
|
32
|
+
messages.each do |input, expect|
|
33
|
+
assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
|
34
|
+
|
35
|
+
expect.each do |key, value|
|
36
|
+
actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
|
37
|
+
assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end # def test_full_valid_message
|
41
|
+
|
42
|
+
def test_valid_message_no_pri
|
43
|
+
messages = {
|
44
|
+
"Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
|
45
|
+
:pri => "13",
|
46
|
+
:timestamp => "Mar 1 15:43:35",
|
47
|
+
:hostname => "snack",
|
48
|
+
:message => "kernel: Kernel logging (proc) stopped."
|
49
|
+
},
|
50
|
+
"Jun 17 03:29:44 1.2.3.4 fancypants" => {
|
51
|
+
:pri => "13",
|
52
|
+
:timestamp => "Jun 17 03:29:44",
|
53
|
+
:hostname => "1.2.3.4",
|
54
|
+
:message => "fancypants"
|
55
|
+
},
|
56
|
+
"Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
|
57
|
+
:pri => "13",
|
58
|
+
:timestamp => "Dec 31 22:00:00",
|
59
|
+
:hostname => "ffe0::1",
|
60
|
+
:message => "something[12345]: hello world",
|
61
|
+
},
|
62
|
+
}
|
63
|
+
|
64
|
+
event = LogPorter::Event.new
|
65
|
+
messages.each do |input, expect|
|
66
|
+
assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
|
67
|
+
|
68
|
+
expect.each do |key, value|
|
69
|
+
actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
|
70
|
+
assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end # def test_valid_message_no_pri
|
74
|
+
|
75
|
+
def test_invalid_message
|
76
|
+
messages = [
|
77
|
+
" Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped.",
|
78
|
+
"<1234>Jun 17 03:29:44 1.2.3.4 fancypants",
|
79
|
+
"<123>Mon, Dec 31 22:00:00 ffe0::1 something[12345]: hello world",
|
80
|
+
]
|
81
|
+
|
82
|
+
event = LogPorter::Event.new
|
83
|
+
messages.each do |input|
|
84
|
+
assert(parse_rfc3164(input, event) == false, "Parse should return false on a invalid message #{input.inspect}")
|
85
|
+
end
|
86
|
+
end # def test_invalid_message
|
87
|
+
end # class TestSyslog3164Parser
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logporter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 0
|
9
|
+
- 1
|
10
|
+
version: 0.0.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Jordan Sissel
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-03-21 00:00:00 -07:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: None yet.
|
23
|
+
email: jordan@loggly.com
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- lib/logporter/protocol/syslog3164.rb
|
32
|
+
- lib/logporter/namespace.rb
|
33
|
+
- lib/logporter/server.rb
|
34
|
+
- lib/logporter/server/defaulthandler.rb
|
35
|
+
- lib/logporter/server/connection.rb
|
36
|
+
- lib/logporter/event.rb
|
37
|
+
- test/syslog.rb
|
38
|
+
has_rdoc: true
|
39
|
+
homepage: https://github.com/loggly/logporter
|
40
|
+
licenses: []
|
41
|
+
|
42
|
+
post_install_message:
|
43
|
+
rdoc_options: []
|
44
|
+
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
- lib
|
48
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
hash: 3
|
54
|
+
segments:
|
55
|
+
- 0
|
56
|
+
version: "0"
|
57
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 3
|
63
|
+
segments:
|
64
|
+
- 0
|
65
|
+
version: "0"
|
66
|
+
requirements: []
|
67
|
+
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 1.5.1
|
70
|
+
signing_key:
|
71
|
+
specification_version: 3
|
72
|
+
summary: logporter - a log server
|
73
|
+
test_files: []
|
74
|
+
|