logporter 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,24 @@
1
+ require "logporter/namespace"
2
+
3
+ class LogPorter::Event
4
+ attr_accessor :pri
5
+ attr_accessor :timestamp
6
+ attr_accessor :hostname
7
+ attr_accessor :message
8
+ attr_accessor :raw
9
+
10
+ # TODO(sissel): Should we include other source information like client
11
+ # address and port?
12
+
13
+ def time_iso8601
14
+ return timestamp.strftime("%Y-%m-%dT%H:%M:%S.") + timestamp.tv_usec.to_s
15
+ end
16
+
17
+ def to_s
18
+ if @raw == true
19
+ return message
20
+ else
21
+ return "<#{pri}>#{time_iso8601} #{hostname} #{message}"
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,4 @@
1
+ module LogPorter
2
+ class Server; end
3
+ module Protocol; end
4
+ end
@@ -0,0 +1,46 @@
1
+ require "logporter/namespace"
2
+ require "time" # for Time.strptime
3
+
4
+ module LogPorter::Protocol::Syslog3164
5
+ def syslog3164_init
6
+ pri = "(?:<(?<pri>[0-9]{1,3})>)?"
7
+ month = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
8
+ day = "(?: [1-9]|[12][0-9]|3[01])"
9
+ hour = "(?:[01][0-9]|2[0-4])"
10
+ minute = "(?:[0-5][0-9])"
11
+ second = "(?:[0-5][0-9])"
12
+
13
+ time = [hour, minute, second].join(":")
14
+
15
+ timestamp = "(?<timestamp>#{month} #{day} #{time})"
16
+ hostname = "(?<hostname>[A-Za-z0-9_.:]+)"
17
+ header = timestamp + " " + hostname
18
+ message = "(?<message>[ -~]+)" # ascii 32 to 126
19
+ re = "^#{pri}#{header} #{message}$"
20
+
21
+ if RUBY_VERSION =~ /^1\.8/
22
+ # Ruby 1.8 doesn't support named captures
23
+ # replace (?<foo> with (
24
+ re = re.gsub(/\(\?<[^>]+>/, "(")
25
+ end
26
+
27
+ @syslog3164_re = Regexp.new(re)
28
+ end
29
+
30
+ def parse_rfc3164(line, event)
31
+ syslog3164_init if !@syslog3164_re
32
+ m = @syslog3164_re.match(line)
33
+ if m
34
+ # RFC3164 section 4.3.3 No PRI or Unidentifiable PRI
35
+ event.pri = m[1] || "13"
36
+
37
+ # TODO(sissel): DateTime is a very slow library, consider alternatives?
38
+ event.timestamp = Time.strptime(m[2], "%b %d %H:%M:%S")
39
+ event.hostname = m[3]
40
+ event.message = m[4]
41
+ return true
42
+ end
43
+ return false
44
+ end # def parse_rfc3164
45
+ end # module Log::Protocol::Syslog3164
46
+
@@ -0,0 +1,113 @@
1
+ require "eventmachine"
2
+ require "logporter/event"
3
+ require "logporter/namespace"
4
+ require "logporter/protocol/syslog3164"
5
+ require "logporter/server"
6
+ require "socket"
7
+
8
+ class LogPorter::Server::Connection < EventMachine::Connection
9
+ include LogPorter::Protocol::Syslog3164
10
+
11
+ def initialize(server)
12
+ @server = server
13
+ end
14
+
15
+ def post_init
16
+ super
17
+
18
+ if @server.network == :tls
19
+ if RUBY_PLATFORM == "java"
20
+ $STDERR.puts "Warning... EventMachine doesn't support TLS on JRuby :("
21
+ end
22
+
23
+ # Calls EventMachine::Connection#start_tls
24
+ start_tls(
25
+ :private_key_file => @server.tls.private_key_file,
26
+ :cert_chain_file => @server.tls.cert_chain_file,
27
+ :verify_peer => @server.tls.verify_peer
28
+ )
29
+ end
30
+
31
+ @count = 0
32
+
33
+ case @server.wire
34
+ when :raw
35
+ class << self
36
+ alias_method :receive_line, :receive_line_raw
37
+ end
38
+ when :syslog
39
+ class << self
40
+ alias_method :receive_line, :receive_line_syslog
41
+ end
42
+ else
43
+ raise "Unsupported protocol #{@server.protocol}"
44
+ end
45
+
46
+ begin
47
+ @client_port, @client_address = Socket.unpack_sockaddr_in(get_peername)
48
+ puts "New client: #{@client_address}:#{@client_port}"
49
+ rescue => e
50
+ p e
51
+ end
52
+ end # def post_init
53
+
54
+ #def ssl_handshake_completed
55
+ # TODO(sissel): validate other pieces of the cert?
56
+ #puts get_peer_cert
57
+ #end
58
+
59
+ def receive_data(data)
60
+ if @server.network == :udp
61
+ client_port, client_address = Socket.unpack_sockaddr_in(get_peername)
62
+ else
63
+ client_port = @client_port
64
+ client_address = @client_address
65
+ end
66
+
67
+ @buffer ||= BufferedTokenizer.new
68
+ @buffer.extract(data).each do |line|
69
+ receive_line(line.chomp, client_address, client_port)
70
+ end
71
+ end
72
+
73
+ def receive_line_raw(line, address, port)
74
+ event = LogPorter::Event.new
75
+
76
+ # TODO(sissel): Look for an alternative to Time#strftime since it is
77
+ # insanely slow.
78
+ event.pri = "13" # RFC3164 says unknown pri == 13.
79
+ event.timestamp = Time.now
80
+ event.hostname = address
81
+ event.message = line
82
+ event.raw = true
83
+
84
+ @server.receive_event(event, address, port)
85
+ stats
86
+ end
87
+
88
+ def receive_line_syslog(line, address, port)
89
+ event = LogPorter::Event.new
90
+ if parse_rfc3164(line, event)
91
+ #elsif parse_rfc5424(line, event)
92
+ else
93
+ # Unknown message format, add syslog headers.
94
+ event.pri = "13" # RFC3164 says unknown pri == 13.
95
+ event.timestamp = Time.now
96
+ event.hostname = address
97
+ event.message = line
98
+ end
99
+
100
+ @server.receive_event(event, address, port)
101
+ stats
102
+ end # def receive_line_syslog
103
+
104
+ def stats
105
+ @start ||= Time.now
106
+ @count += 1
107
+ if @count % 50000 == 0
108
+ puts "Rate: #{@count / (Time.now - @start)}"
109
+ @start = Time.now
110
+ @count = 0
111
+ end
112
+ end # def stats
113
+ end
@@ -0,0 +1,9 @@
1
+
2
+
3
+ require "logporter/namespace"
4
+
5
+ class LogPorter::Server::DefaultHandler
6
+ def receive_event(event, server, client_addr, client_port)
7
+ puts "#{client_addr}:#{client_port}(#{server.network}/#{server.wire}) => #{event}"
8
+ end # def receive_event
9
+ end # class Log::Server::DefaultHandler
@@ -0,0 +1,109 @@
1
+ require "logporter/namespace"
2
+ require "logporter/server/connection"
3
+ require "logporter/server/defaulthandler"
4
+
5
+ class LogPorter::Server
6
+
7
+ # Create a new class called 'TLSConfig' which simply acts as a data structure.
8
+ TLSConfig = Struct.new :private_key_file, :cert_chain_file, :verify_peer
9
+
10
+ # The port we are listening on
11
+ attr_reader :port
12
+
13
+ # TLS options, only meaningful if @network == :tls
14
+ attr_reader :tls
15
+
16
+ # The network layer, :tcp, :udp, or :tls
17
+ attr_reader :network
18
+
19
+ # The wire format (syslog, raw, etc)
20
+ attr_reader :wire
21
+
22
+ # Arbitrary attributes for this server. You can store whatever you want here.
23
+ # This is a hash
24
+ attr_reader :attributes
25
+
26
+ # Create a new server to listen with
27
+ # 'options' is a hash of:
28
+ #
29
+ # :net => the network layer to use (:udp, :tcp, :tls)
30
+ # :port => the port to listen on
31
+ # :wire => the wire format (:raw, :syslog)
32
+ # :handler => the handler instance. Must respond to 'receive_event'
33
+ def initialize(options)
34
+ @network = options[:net]
35
+ @port = options.delete(:port) || 514
36
+ @wire = options.delete(:wire) || :raw
37
+ @handler = options.delete(:handler) || LogPorter::Server::DefaultHandler.new
38
+ @attributes = options.delete(:attributes) || Hash.new
39
+
40
+ if @network == :tls
41
+ @tls = TLSConfig.new
42
+ @tls.private_key_file = options[:private_key]
43
+ @tls.cert_chain_file = options[:certificate_chain]
44
+ @tls.verify_peer = options[:verify_peer] || false
45
+ else
46
+ @tls = nil
47
+ end
48
+ end # def initialize
49
+
50
+ # start the server
51
+ public
52
+ def start
53
+ # We use next_tick here in case you are invoking this method from outside
54
+ # of EventMachine; this allows you to do this:
55
+ #
56
+ # s = LogPorter::server.new ...
57
+ # s.start
58
+ #
59
+ # EventMachine.run()
60
+ EventMachine.next_tick do
61
+ puts "Starting #{@network}/#{@port}"
62
+ begin
63
+ case @network
64
+ when :udp; start_udp_server
65
+ when :tcp; start_tcp_server
66
+ when :tls; start_tcp_server # tls is handled by tcp.
67
+ else
68
+ raise "Unknown network '#{@network}' expected :udp, :tcp, or :tls"
69
+ end
70
+ rescue => e
71
+ if @handler.respond_to?(:receive_exception)
72
+ @handler.receive_exception(e)
73
+ else
74
+ raise e
75
+ end
76
+ end
77
+ end
78
+ end # def start
79
+
80
+ private
81
+ def start_udp_server
82
+ @socket = EventMachine::open_datagram_socket "0.0.0.0", @port,
83
+ LogPorter::Server::Connection, self
84
+ end # def start_udp
85
+
86
+ private
87
+ def start_tcp_server
88
+ @socket = EventMachine::start_server "0.0.0.0", @port,
89
+ LogPorter::Server::Connection, self
90
+ end # def start_tcp
91
+
92
+ # This method is invoked by LogPorter::Server::Connection
93
+ public
94
+ def receive_event(event, client_addr, client_port)
95
+ @handler.receive_event(event, self, client_addr, client_port)
96
+ end
97
+
98
+ public
99
+ def stop
100
+ case @network
101
+ when :tcp
102
+ EventMachine::stop_server(@socket)
103
+ when :tls
104
+ EventMachine::stop_server(@socket)
105
+ when :udp
106
+ @socket.close_connection(true)
107
+ end
108
+ end # def stop
109
+ end # class LogPorter::Server
data/test/syslog.rb ADDED
@@ -0,0 +1,87 @@
1
+
2
+ require "test/unit"
3
+ require "logporter/protocol/syslog3164"
4
+ require "logporter/event"
5
+
6
+ class TestSyslog3164Parser < Test::Unit::TestCase
7
+ include LogPorter::Protocol::Syslog3164
8
+
9
+ def test_full_valid_message
10
+ messages = {
11
+ "<12>Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
12
+ :pri => "12",
13
+ :timestamp => "Mar 1 15:43:35",
14
+ :hostname => "snack",
15
+ :message => "kernel: Kernel logging (proc) stopped."
16
+ },
17
+ "<1>Jun 17 03:29:44 1.2.3.4 fancypants" => {
18
+ :pri => "1",
19
+ :timestamp => "Jun 17 03:29:44",
20
+ :hostname => "1.2.3.4",
21
+ :message => "fancypants"
22
+ },
23
+ "<100>Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
24
+ :pri => "100",
25
+ :timestamp => "Dec 31 22:00:00",
26
+ :hostname => "ffe0::1",
27
+ :message => "something[12345]: hello world",
28
+ },
29
+ }
30
+
31
+ event = LogPorter::Event.new
32
+ messages.each do |input, expect|
33
+ assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
34
+
35
+ expect.each do |key, value|
36
+ actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
37
+ assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
38
+ end
39
+ end
40
+ end # def test_full_valid_message
41
+
42
+ def test_valid_message_no_pri
43
+ messages = {
44
+ "Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
45
+ :pri => "13",
46
+ :timestamp => "Mar 1 15:43:35",
47
+ :hostname => "snack",
48
+ :message => "kernel: Kernel logging (proc) stopped."
49
+ },
50
+ "Jun 17 03:29:44 1.2.3.4 fancypants" => {
51
+ :pri => "13",
52
+ :timestamp => "Jun 17 03:29:44",
53
+ :hostname => "1.2.3.4",
54
+ :message => "fancypants"
55
+ },
56
+ "Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
57
+ :pri => "13",
58
+ :timestamp => "Dec 31 22:00:00",
59
+ :hostname => "ffe0::1",
60
+ :message => "something[12345]: hello world",
61
+ },
62
+ }
63
+
64
+ event = LogPorter::Event.new
65
+ messages.each do |input, expect|
66
+ assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
67
+
68
+ expect.each do |key, value|
69
+ actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
70
+ assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
71
+ end
72
+ end
73
+ end # def test_valid_message_no_pri
74
+
75
+ def test_invalid_message
76
+ messages = [
77
+ " Mar 1 15:43:35 snack kernel: Kernel logging (proc) stopped.",
78
+ "<1234>Jun 17 03:29:44 1.2.3.4 fancypants",
79
+ "<123>Mon, Dec 31 22:00:00 ffe0::1 something[12345]: hello world",
80
+ ]
81
+
82
+ event = LogPorter::Event.new
83
+ messages.each do |input|
84
+ assert(parse_rfc3164(input, event) == false, "Parse should return false on a invalid message #{input.inspect}")
85
+ end
86
+ end # def test_invalid_message
87
+ end # class TestSyslog3164Parser
metadata ADDED
@@ -0,0 +1,74 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logporter
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 1
10
+ version: 0.0.1
11
+ platform: ruby
12
+ authors:
13
+ - Jordan Sissel
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2011-03-21 00:00:00 -07:00
19
+ default_executable:
20
+ dependencies: []
21
+
22
+ description: None yet.
23
+ email: jordan@loggly.com
24
+ executables: []
25
+
26
+ extensions: []
27
+
28
+ extra_rdoc_files: []
29
+
30
+ files:
31
+ - lib/logporter/protocol/syslog3164.rb
32
+ - lib/logporter/namespace.rb
33
+ - lib/logporter/server.rb
34
+ - lib/logporter/server/defaulthandler.rb
35
+ - lib/logporter/server/connection.rb
36
+ - lib/logporter/event.rb
37
+ - test/syslog.rb
38
+ has_rdoc: true
39
+ homepage: https://github.com/loggly/logporter
40
+ licenses: []
41
+
42
+ post_install_message:
43
+ rdoc_options: []
44
+
45
+ require_paths:
46
+ - lib
47
+ - lib
48
+ required_ruby_version: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ hash: 3
54
+ segments:
55
+ - 0
56
+ version: "0"
57
+ required_rubygems_version: !ruby/object:Gem::Requirement
58
+ none: false
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ hash: 3
63
+ segments:
64
+ - 0
65
+ version: "0"
66
+ requirements: []
67
+
68
+ rubyforge_project:
69
+ rubygems_version: 1.5.1
70
+ signing_key:
71
+ specification_version: 3
72
+ summary: logporter - a log server
73
+ test_files: []
74
+