logporter 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/logporter/event.rb +24 -0
- data/lib/logporter/namespace.rb +4 -0
- data/lib/logporter/protocol/syslog3164.rb +46 -0
- data/lib/logporter/server/connection.rb +113 -0
- data/lib/logporter/server/defaulthandler.rb +9 -0
- data/lib/logporter/server.rb +109 -0
- data/test/syslog.rb +87 -0
- metadata +74 -0
@@ -0,0 +1,24 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
|
3
|
+
class LogPorter::Event
|
4
|
+
attr_accessor :pri
|
5
|
+
attr_accessor :timestamp
|
6
|
+
attr_accessor :hostname
|
7
|
+
attr_accessor :message
|
8
|
+
attr_accessor :raw
|
9
|
+
|
10
|
+
# TODO(sissel): Should we include other source information like client
|
11
|
+
# address and port?
|
12
|
+
|
13
|
+
def time_iso8601
|
14
|
+
return timestamp.strftime("%Y-%m-%dT%H:%M:%S.") + timestamp.tv_usec.to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
if @raw == true
|
19
|
+
return message
|
20
|
+
else
|
21
|
+
return "<#{pri}>#{time_iso8601} #{hostname} #{message}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
require "time" # for Time.strptime
|
3
|
+
|
4
|
+
module LogPorter::Protocol::Syslog3164
|
5
|
+
def syslog3164_init
|
6
|
+
pri = "(?:<(?<pri>[0-9]{1,3})>)?"
|
7
|
+
month = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
|
8
|
+
day = "(?: [1-9]|[12][0-9]|3[01])"
|
9
|
+
hour = "(?:[01][0-9]|2[0-4])"
|
10
|
+
minute = "(?:[0-5][0-9])"
|
11
|
+
second = "(?:[0-5][0-9])"
|
12
|
+
|
13
|
+
time = [hour, minute, second].join(":")
|
14
|
+
|
15
|
+
timestamp = "(?<timestamp>#{month} #{day} #{time})"
|
16
|
+
hostname = "(?<hostname>[A-Za-z0-9_.:]+)"
|
17
|
+
header = timestamp + " " + hostname
|
18
|
+
message = "(?<message>[ -~]+)" # ascii 32 to 126
|
19
|
+
re = "^#{pri}#{header} #{message}$"
|
20
|
+
|
21
|
+
if RUBY_VERSION =~ /^1\.8/
|
22
|
+
# Ruby 1.8 doesn't support named captures
|
23
|
+
# replace (?<foo> with (
|
24
|
+
re = re.gsub(/\(\?<[^>]+>/, "(")
|
25
|
+
end
|
26
|
+
|
27
|
+
@syslog3164_re = Regexp.new(re)
|
28
|
+
end
|
29
|
+
|
30
|
+
def parse_rfc3164(line, event)
|
31
|
+
syslog3164_init if !@syslog3164_re
|
32
|
+
m = @syslog3164_re.match(line)
|
33
|
+
if m
|
34
|
+
# RFC3164 section 4.3.3 No PRI or Unidentifiable PRI
|
35
|
+
event.pri = m[1] || "13"
|
36
|
+
|
37
|
+
# TODO(sissel): DateTime is a very slow library, consider alternatives?
|
38
|
+
event.timestamp = Time.strptime(m[2], "%b %d %H:%M:%S")
|
39
|
+
event.hostname = m[3]
|
40
|
+
event.message = m[4]
|
41
|
+
return true
|
42
|
+
end
|
43
|
+
return false
|
44
|
+
end # def parse_rfc3164
|
45
|
+
end # module Log::Protocol::Syslog3164
|
46
|
+
|
@@ -0,0 +1,113 @@
|
|
1
|
+
require "eventmachine"
|
2
|
+
require "logporter/event"
|
3
|
+
require "logporter/namespace"
|
4
|
+
require "logporter/protocol/syslog3164"
|
5
|
+
require "logporter/server"
|
6
|
+
require "socket"
|
7
|
+
|
8
|
+
class LogPorter::Server::Connection < EventMachine::Connection
|
9
|
+
include LogPorter::Protocol::Syslog3164
|
10
|
+
|
11
|
+
def initialize(server)
|
12
|
+
@server = server
|
13
|
+
end
|
14
|
+
|
15
|
+
def post_init
|
16
|
+
super
|
17
|
+
|
18
|
+
if @server.network == :tls
|
19
|
+
if RUBY_PLATFORM == "java"
|
20
|
+
$STDERR.puts "Warning... EventMachine doesn't support TLS on JRuby :("
|
21
|
+
end
|
22
|
+
|
23
|
+
# Calls EventMachine::Connection#start_tls
|
24
|
+
start_tls(
|
25
|
+
:private_key_file => @server.tls.private_key_file,
|
26
|
+
:cert_chain_file => @server.tls.cert_chain_file,
|
27
|
+
:verify_peer => @server.tls.verify_peer
|
28
|
+
)
|
29
|
+
end
|
30
|
+
|
31
|
+
@count = 0
|
32
|
+
|
33
|
+
case @server.wire
|
34
|
+
when :raw
|
35
|
+
class << self
|
36
|
+
alias_method :receive_line, :receive_line_raw
|
37
|
+
end
|
38
|
+
when :syslog
|
39
|
+
class << self
|
40
|
+
alias_method :receive_line, :receive_line_syslog
|
41
|
+
end
|
42
|
+
else
|
43
|
+
raise "Unsupported protocol #{@server.protocol}"
|
44
|
+
end
|
45
|
+
|
46
|
+
begin
|
47
|
+
@client_port, @client_address = Socket.unpack_sockaddr_in(get_peername)
|
48
|
+
puts "New client: #{@client_address}:#{@client_port}"
|
49
|
+
rescue => e
|
50
|
+
p e
|
51
|
+
end
|
52
|
+
end # def post_init
|
53
|
+
|
54
|
+
#def ssl_handshake_completed
|
55
|
+
# TODO(sissel): validate other pieces of the cert?
|
56
|
+
#puts get_peer_cert
|
57
|
+
#end
|
58
|
+
|
59
|
+
def receive_data(data)
|
60
|
+
if @server.network == :udp
|
61
|
+
client_port, client_address = Socket.unpack_sockaddr_in(get_peername)
|
62
|
+
else
|
63
|
+
client_port = @client_port
|
64
|
+
client_address = @client_address
|
65
|
+
end
|
66
|
+
|
67
|
+
@buffer ||= BufferedTokenizer.new
|
68
|
+
@buffer.extract(data).each do |line|
|
69
|
+
receive_line(line.chomp, client_address, client_port)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def receive_line_raw(line, address, port)
|
74
|
+
event = LogPorter::Event.new
|
75
|
+
|
76
|
+
# TODO(sissel): Look for an alternative to Time#strftime since it is
|
77
|
+
# insanely slow.
|
78
|
+
event.pri = "13" # RFC3164 says unknown pri == 13.
|
79
|
+
event.timestamp = Time.now
|
80
|
+
event.hostname = address
|
81
|
+
event.message = line
|
82
|
+
event.raw = true
|
83
|
+
|
84
|
+
@server.receive_event(event, address, port)
|
85
|
+
stats
|
86
|
+
end
|
87
|
+
|
88
|
+
def receive_line_syslog(line, address, port)
|
89
|
+
event = LogPorter::Event.new
|
90
|
+
if parse_rfc3164(line, event)
|
91
|
+
#elsif parse_rfc5424(line, event)
|
92
|
+
else
|
93
|
+
# Unknown message format, add syslog headers.
|
94
|
+
event.pri = "13" # RFC3164 says unknown pri == 13.
|
95
|
+
event.timestamp = Time.now
|
96
|
+
event.hostname = address
|
97
|
+
event.message = line
|
98
|
+
end
|
99
|
+
|
100
|
+
@server.receive_event(event, address, port)
|
101
|
+
stats
|
102
|
+
end # def receive_line_syslog
|
103
|
+
|
104
|
+
def stats
|
105
|
+
@start ||= Time.now
|
106
|
+
@count += 1
|
107
|
+
if @count % 50000 == 0
|
108
|
+
puts "Rate: #{@count / (Time.now - @start)}"
|
109
|
+
@start = Time.now
|
110
|
+
@count = 0
|
111
|
+
end
|
112
|
+
end # def stats
|
113
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
require "logporter/namespace"
|
4
|
+
|
5
|
+
class LogPorter::Server::DefaultHandler
|
6
|
+
def receive_event(event, server, client_addr, client_port)
|
7
|
+
puts "#{client_addr}:#{client_port}(#{server.network}/#{server.wire}) => #{event}"
|
8
|
+
end # def receive_event
|
9
|
+
end # class Log::Server::DefaultHandler
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require "logporter/namespace"
|
2
|
+
require "logporter/server/connection"
|
3
|
+
require "logporter/server/defaulthandler"
|
4
|
+
|
5
|
+
class LogPorter::Server
|
6
|
+
|
7
|
+
# Create a new class called 'TLSConfig' which simply acts as a data structure.
|
8
|
+
TLSConfig = Struct.new :private_key_file, :cert_chain_file, :verify_peer
|
9
|
+
|
10
|
+
# The port we are listening on
|
11
|
+
attr_reader :port
|
12
|
+
|
13
|
+
# TLS options, only meaningful if @network == :tls
|
14
|
+
attr_reader :tls
|
15
|
+
|
16
|
+
# The network layer, :tcp, :udp, or :tls
|
17
|
+
attr_reader :network
|
18
|
+
|
19
|
+
# The wire format (syslog, raw, etc)
|
20
|
+
attr_reader :wire
|
21
|
+
|
22
|
+
# Arbitrary attributes for this server. You can store whatever you want here.
|
23
|
+
# This is a hash
|
24
|
+
attr_reader :attributes
|
25
|
+
|
26
|
+
# Create a new server to listen with
|
27
|
+
# 'options' is a hash of:
|
28
|
+
#
|
29
|
+
# :net => the network layer to use (:udp, :tcp, :tls)
|
30
|
+
# :port => the port to listen on
|
31
|
+
# :wire => the wire format (:raw, :syslog)
|
32
|
+
# :handler => the handler instance. Must respond to 'receive_event'
|
33
|
+
def initialize(options)
|
34
|
+
@network = options[:net]
|
35
|
+
@port = options.delete(:port) || 514
|
36
|
+
@wire = options.delete(:wire) || :raw
|
37
|
+
@handler = options.delete(:handler) || LogPorter::Server::DefaultHandler.new
|
38
|
+
@attributes = options.delete(:attributes) || Hash.new
|
39
|
+
|
40
|
+
if @network == :tls
|
41
|
+
@tls = TLSConfig.new
|
42
|
+
@tls.private_key_file = options[:private_key]
|
43
|
+
@tls.cert_chain_file = options[:certificate_chain]
|
44
|
+
@tls.verify_peer = options[:verify_peer] || false
|
45
|
+
else
|
46
|
+
@tls = nil
|
47
|
+
end
|
48
|
+
end # def initialize
|
49
|
+
|
50
|
+
# start the server
|
51
|
+
public
|
52
|
+
def start
|
53
|
+
# We use next_tick here in case you are invoking this method from outside
|
54
|
+
# of EventMachine; this allows you to do this:
|
55
|
+
#
|
56
|
+
# s = LogPorter::server.new ...
|
57
|
+
# s.start
|
58
|
+
#
|
59
|
+
# EventMachine.run()
|
60
|
+
EventMachine.next_tick do
|
61
|
+
puts "Starting #{@network}/#{@port}"
|
62
|
+
begin
|
63
|
+
case @network
|
64
|
+
when :udp; start_udp_server
|
65
|
+
when :tcp; start_tcp_server
|
66
|
+
when :tls; start_tcp_server # tls is handled by tcp.
|
67
|
+
else
|
68
|
+
raise "Unknown network '#{@network}' expected :udp, :tcp, or :tls"
|
69
|
+
end
|
70
|
+
rescue => e
|
71
|
+
if @handler.respond_to?(:receive_exception)
|
72
|
+
@handler.receive_exception(e)
|
73
|
+
else
|
74
|
+
raise e
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end # def start
|
79
|
+
|
80
|
+
private
|
81
|
+
def start_udp_server
|
82
|
+
@socket = EventMachine::open_datagram_socket "0.0.0.0", @port,
|
83
|
+
LogPorter::Server::Connection, self
|
84
|
+
end # def start_udp
|
85
|
+
|
86
|
+
private
|
87
|
+
def start_tcp_server
|
88
|
+
@socket = EventMachine::start_server "0.0.0.0", @port,
|
89
|
+
LogPorter::Server::Connection, self
|
90
|
+
end # def start_tcp
|
91
|
+
|
92
|
+
# This method is invoked by LogPorter::Server::Connection
|
93
|
+
public
|
94
|
+
def receive_event(event, client_addr, client_port)
|
95
|
+
@handler.receive_event(event, self, client_addr, client_port)
|
96
|
+
end
|
97
|
+
|
98
|
+
public
|
99
|
+
def stop
|
100
|
+
case @network
|
101
|
+
when :tcp
|
102
|
+
EventMachine::stop_server(@socket)
|
103
|
+
when :tls
|
104
|
+
EventMachine::stop_server(@socket)
|
105
|
+
when :udp
|
106
|
+
@socket.close_connection(true)
|
107
|
+
end
|
108
|
+
end # def stop
|
109
|
+
end # class LogPorter::Server
|
data/test/syslog.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
|
2
|
+
require "test/unit"
|
3
|
+
require "logporter/protocol/syslog3164"
|
4
|
+
require "logporter/event"
|
5
|
+
|
6
|
+
class TestSyslog3164Parser < Test::Unit::TestCase
|
7
|
+
include LogPorter::Protocol::Syslog3164
|
8
|
+
|
9
|
+
def test_full_valid_message
|
10
|
+
messages = {
|
11
|
+
"<12>Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
|
12
|
+
:pri => "12",
|
13
|
+
:timestamp => "Mar 1 15:43:35",
|
14
|
+
:hostname => "snack",
|
15
|
+
:message => "kernel: Kernel logging (proc) stopped."
|
16
|
+
},
|
17
|
+
"<1>Jun 17 03:29:44 1.2.3.4 fancypants" => {
|
18
|
+
:pri => "1",
|
19
|
+
:timestamp => "Jun 17 03:29:44",
|
20
|
+
:hostname => "1.2.3.4",
|
21
|
+
:message => "fancypants"
|
22
|
+
},
|
23
|
+
"<100>Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
|
24
|
+
:pri => "100",
|
25
|
+
:timestamp => "Dec 31 22:00:00",
|
26
|
+
:hostname => "ffe0::1",
|
27
|
+
:message => "something[12345]: hello world",
|
28
|
+
},
|
29
|
+
}
|
30
|
+
|
31
|
+
event = LogPorter::Event.new
|
32
|
+
messages.each do |input, expect|
|
33
|
+
assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
|
34
|
+
|
35
|
+
expect.each do |key, value|
|
36
|
+
actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
|
37
|
+
assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end # def test_full_valid_message
|
41
|
+
|
42
|
+
def test_valid_message_no_pri
|
43
|
+
messages = {
|
44
|
+
"Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
|
45
|
+
:pri => "13",
|
46
|
+
:timestamp => "Mar 1 15:43:35",
|
47
|
+
:hostname => "snack",
|
48
|
+
:message => "kernel: Kernel logging (proc) stopped."
|
49
|
+
},
|
50
|
+
"Jun 17 03:29:44 1.2.3.4 fancypants" => {
|
51
|
+
:pri => "13",
|
52
|
+
:timestamp => "Jun 17 03:29:44",
|
53
|
+
:hostname => "1.2.3.4",
|
54
|
+
:message => "fancypants"
|
55
|
+
},
|
56
|
+
"Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
|
57
|
+
:pri => "13",
|
58
|
+
:timestamp => "Dec 31 22:00:00",
|
59
|
+
:hostname => "ffe0::1",
|
60
|
+
:message => "something[12345]: hello world",
|
61
|
+
},
|
62
|
+
}
|
63
|
+
|
64
|
+
event = LogPorter::Event.new
|
65
|
+
messages.each do |input, expect|
|
66
|
+
assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
|
67
|
+
|
68
|
+
expect.each do |key, value|
|
69
|
+
actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
|
70
|
+
assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end # def test_valid_message_no_pri
|
74
|
+
|
75
|
+
def test_invalid_message
|
76
|
+
messages = [
|
77
|
+
" Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped.",
|
78
|
+
"<1234>Jun 17 03:29:44 1.2.3.4 fancypants",
|
79
|
+
"<123>Mon, Dec 31 22:00:00 ffe0::1 something[12345]: hello world",
|
80
|
+
]
|
81
|
+
|
82
|
+
event = LogPorter::Event.new
|
83
|
+
messages.each do |input|
|
84
|
+
assert(parse_rfc3164(input, event) == false, "Parse should return false on a invalid message #{input.inspect}")
|
85
|
+
end
|
86
|
+
end # def test_invalid_message
|
87
|
+
end # class TestSyslog3164Parser
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logporter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 0
|
9
|
+
- 1
|
10
|
+
version: 0.0.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Jordan Sissel
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-03-21 00:00:00 -07:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: None yet.
|
23
|
+
email: jordan@loggly.com
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- lib/logporter/protocol/syslog3164.rb
|
32
|
+
- lib/logporter/namespace.rb
|
33
|
+
- lib/logporter/server.rb
|
34
|
+
- lib/logporter/server/defaulthandler.rb
|
35
|
+
- lib/logporter/server/connection.rb
|
36
|
+
- lib/logporter/event.rb
|
37
|
+
- test/syslog.rb
|
38
|
+
has_rdoc: true
|
39
|
+
homepage: https://github.com/loggly/logporter
|
40
|
+
licenses: []
|
41
|
+
|
42
|
+
post_install_message:
|
43
|
+
rdoc_options: []
|
44
|
+
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
- lib
|
48
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
hash: 3
|
54
|
+
segments:
|
55
|
+
- 0
|
56
|
+
version: "0"
|
57
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 3
|
63
|
+
segments:
|
64
|
+
- 0
|
65
|
+
version: "0"
|
66
|
+
requirements: []
|
67
|
+
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 1.5.1
|
70
|
+
signing_key:
|
71
|
+
specification_version: 3
|
72
|
+
summary: logporter - a log server
|
73
|
+
test_files: []
|
74
|
+
|