logporter 0.0.1

data/lib/logporter/event.rb

@@ -0,0 +1,24 @@
+require "logporter/namespace"
+
+class LogPorter::Event
+  attr_accessor :pri
+  attr_accessor :timestamp
+  attr_accessor :hostname
+  attr_accessor :message
+  attr_accessor :raw
+
+  # TODO(sissel): Should we include other source information like client
+  # address and port?
+
+  def time_iso8601
+    return timestamp.strftime("%Y-%m-%dT%H:%M:%S.") + timestamp.tv_usec.to_s
+  end
+
+  def to_s
+    if @raw == true
+      return message
+    else
+      return "<#{pri}>#{time_iso8601} #{hostname} #{message}"
+    end
+  end
+end

data/lib/logporter/namespace.rb

@@ -0,0 +1,4 @@
+module LogPorter
+  class Server; end
+  module Protocol; end
+end

data/lib/logporter/protocol/syslog3164.rb

@@ -0,0 +1,46 @@
+require "logporter/namespace"
+require "time" # for Time.strptime
+
+module LogPorter::Protocol::Syslog3164
+  def syslog3164_init
+    pri = "(?:<(?<pri>[0-9]{1,3})>)?"
+    month = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
+    day = "(?: [1-9]|[12][0-9]|3[01])"
+    hour = "(?:[01][0-9]|2[0-4])"
+    minute = "(?:[0-5][0-9])"
+    second = "(?:[0-5][0-9])"
+
+    time = [hour, minute, second].join(":")
+
+    timestamp = "(?<timestamp>#{month} #{day} #{time})"
+    hostname = "(?<hostname>[A-Za-z0-9_.:]+)"
+    header = timestamp + " " + hostname
+    message = "(?<message>[ -~]+)"  # ascii 32 to 126
+    re = "^#{pri}#{header} #{message}$"
+
+    if RUBY_VERSION =~ /^1\.8/
+      # Ruby 1.8 doesn't support named captures
+      # replace (?<foo> with (
+      re = re.gsub(/\(\?<[^>]+>/, "(")
+    end
+
+    @syslog3164_re = Regexp.new(re)
+  end
+
+  def parse_rfc3164(line, event)
+    syslog3164_init if !@syslog3164_re
+    m = @syslog3164_re.match(line)
+    if m
+      # RFC3164 section 4.3.3 No PRI or Unidentifiable PRI
+      event.pri = m[1] || "13"
+
+      # TODO(sissel): DateTime is a very slow library, consider alternatives?
+      event.timestamp = Time.strptime(m[2], "%b %d %H:%M:%S")
+      event.hostname = m[3]
+      event.message = m[4]
+      return true
+    end
+    return false
+  end # def parse_rfc3164
+end # module Log::Protocol::Syslog3164
+

data/lib/logporter/server/connection.rb

@@ -0,0 +1,113 @@
+require "eventmachine"
+require "logporter/event"
+require "logporter/namespace"
+require "logporter/protocol/syslog3164"
+require "logporter/server"
+require "socket"
+
+class LogPorter::Server::Connection < EventMachine::Connection
+  include LogPorter::Protocol::Syslog3164
+ 
+  def initialize(server)
+    @server = server
+  end
+
+  def post_init
+    super
+
+    if @server.network == :tls
+      if RUBY_PLATFORM == "java"
+        $STDERR.puts "Warning... EventMachine doesn't support TLS on JRuby :("
+      end
+
+      # Calls EventMachine::Connection#start_tls
+      start_tls(
+        :private_key_file => @server.tls.private_key_file,
+        :cert_chain_file => @server.tls.cert_chain_file,
+        :verify_peer => @server.tls.verify_peer
+      )
+    end
+    
+    @count = 0
+
+    case @server.wire
+      when :raw
+        class << self
+          alias_method :receive_line, :receive_line_raw
+        end
+      when :syslog
+        class << self
+          alias_method :receive_line, :receive_line_syslog
+        end
+      else
+        raise "Unsupported protocol #{@server.protocol}"
+    end
+
+    begin
+      @client_port, @client_address = Socket.unpack_sockaddr_in(get_peername)
+      puts "New client: #{@client_address}:#{@client_port}"
+    rescue => e
+      p e
+    end
+  end # def post_init
+
+  #def ssl_handshake_completed
+    # TODO(sissel): validate other pieces of the cert?
+    #puts get_peer_cert
+  #end
+
+  def receive_data(data)
+    if @server.network == :udp
+      client_port, client_address = Socket.unpack_sockaddr_in(get_peername)
+    else
+      client_port = @client_port
+      client_address = @client_address
+    end
+
+    @buffer ||= BufferedTokenizer.new
+    @buffer.extract(data).each do |line|
+      receive_line(line.chomp, client_address, client_port)
+    end
+  end
+
+  def receive_line_raw(line, address, port)
+    event = LogPorter::Event.new
+
+    # TODO(sissel): Look for an alternative to Time#strftime since it is
+    # insanely slow.
+    event.pri = "13" # RFC3164 says unknown pri == 13.
+    event.timestamp = Time.now
+    event.hostname = address
+    event.message = line
+    event.raw = true
+
+    @server.receive_event(event, address, port)
+    stats
+  end
+
+  def receive_line_syslog(line, address, port)
+    event = LogPorter::Event.new
+    if parse_rfc3164(line, event)
+    #elsif parse_rfc5424(line, event)
+    else 
+      # Unknown message format, add syslog headers.
+      event.pri = "13" # RFC3164 says unknown pri == 13.
+      event.timestamp = Time.now
+      event.hostname = address
+      event.message = line
+    end
+
+    @server.receive_event(event, address, port)
+    stats
+  end # def receive_line_syslog
+
+  def stats
+    @start ||= Time.now
+    @count += 1
+    if @count % 50000 == 0
+      puts "Rate: #{@count / (Time.now - @start)}"
+      @start = Time.now
+      @count = 0
+    end
+  end # def stats
+end

data/lib/logporter/server/defaulthandler.rb

@@ -0,0 +1,9 @@
+
+
+require "logporter/namespace"
+
+class LogPorter::Server::DefaultHandler 
+  def receive_event(event, server, client_addr, client_port)
+    puts "#{client_addr}:#{client_port}(#{server.network}/#{server.wire}) => #{event}"
+  end # def receive_event
+end # class Log::Server::DefaultHandler 

data/lib/logporter/server.rb

@@ -0,0 +1,109 @@
+require "logporter/namespace"
+require "logporter/server/connection"
+require "logporter/server/defaulthandler"
+
+class LogPorter::Server
+
+  # Create a new class called 'TLSConfig' which simply acts as a data structure.
+  TLSConfig = Struct.new :private_key_file, :cert_chain_file, :verify_peer
+
+  # The port we are listening on
+  attr_reader :port
+
+  # TLS options, only meaningful if @network == :tls
+  attr_reader :tls
+
+  # The network layer, :tcp, :udp, or :tls
+  attr_reader :network
+
+  # The wire format (syslog, raw, etc)
+  attr_reader :wire
+
+  # Arbitrary attributes for this server. You can store whatever you want here.
+  # This is a hash
+  attr_reader :attributes
+
+  # Create a new server to listen with
+  # 'options' is a hash of:
+  #
+  #   :net => the network layer to use (:udp, :tcp, :tls)
+  #   :port => the port to listen on
+  #   :wire => the wire format (:raw, :syslog)
+  #   :handler => the handler instance. Must respond to 'receive_event'
+  def initialize(options)
+    @network = options[:net]
+    @port = options.delete(:port) || 514
+    @wire = options.delete(:wire) || :raw
+    @handler = options.delete(:handler) || LogPorter::Server::DefaultHandler.new
+    @attributes = options.delete(:attributes) || Hash.new
+
+    if @network == :tls
+      @tls = TLSConfig.new
+      @tls.private_key_file = options[:private_key]
+      @tls.cert_chain_file = options[:certificate_chain]
+      @tls.verify_peer = options[:verify_peer] || false
+    else
+      @tls = nil
+    end
+  end # def initialize
+
+  # start the server
+  public
+  def start
+    # We use next_tick here in case you are invoking this method from outside
+    # of EventMachine; this allows you to do this:
+    #
+    #   s = LogPorter::server.new ...
+    #   s.start
+    #
+    #   EventMachine.run()
+    EventMachine.next_tick do
+      puts "Starting #{@network}/#{@port}"
+      begin
+        case @network
+          when :udp; start_udp_server
+          when :tcp; start_tcp_server
+          when :tls; start_tcp_server # tls is handled by tcp.
+          else
+            raise "Unknown network '#{@network}' expected :udp, :tcp, or :tls"
+        end
+      rescue => e
+        if @handler.respond_to?(:receive_exception)
+          @handler.receive_exception(e)
+        else
+          raise e
+        end
+      end
+    end
+  end # def start
+
+  private
+  def start_udp_server
+    @socket = EventMachine::open_datagram_socket "0.0.0.0", @port,
+      LogPorter::Server::Connection, self
+  end # def start_udp
+
+  private
+  def start_tcp_server
+    @socket = EventMachine::start_server "0.0.0.0", @port,
+      LogPorter::Server::Connection, self
+  end # def start_tcp
+
+  # This method is invoked by LogPorter::Server::Connection
+  public
+  def receive_event(event, client_addr, client_port)
+    @handler.receive_event(event, self, client_addr, client_port)
+  end
+
+  public
+  def stop
+    case @network
+      when :tcp
+        EventMachine::stop_server(@socket)
+      when :tls
+        EventMachine::stop_server(@socket)
+      when :udp
+        @socket.close_connection(true)
+    end
+  end # def stop
+end # class LogPorter::Server

data/test/syslog.rb

@@ -0,0 +1,87 @@
+
+require "test/unit"
+require "logporter/protocol/syslog3164"
+require "logporter/event"
+
+class TestSyslog3164Parser < Test::Unit::TestCase
+  include LogPorter::Protocol::Syslog3164
+
+  def test_full_valid_message
+    messages = {
+      "<12>Mar  1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
+        :pri => "12",
+        :timestamp => "Mar  1 15:43:35",
+        :hostname => "snack",
+        :message => "kernel: Kernel logging (proc) stopped."
+      },
+      "<1>Jun 17 03:29:44 1.2.3.4 fancypants" => {
+        :pri => "1",
+        :timestamp => "Jun 17 03:29:44",
+        :hostname => "1.2.3.4",
+        :message => "fancypants"
+      },
+      "<100>Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
+        :pri => "100",
+        :timestamp => "Dec 31 22:00:00",
+        :hostname => "ffe0::1",
+        :message => "something[12345]: hello world",
+      },
+    }
+
+    event = LogPorter::Event.new
+    messages.each do |input, expect|
+      assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
+
+      expect.each do |key, value|
+        actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
+        assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
+      end
+    end
+  end # def test_full_valid_message
+
+  def test_valid_message_no_pri
+    messages = {
+      "Mar  1 15:43:35 snack kernel: Kernel logging (proc) stopped." => {
+        :pri => "13",
+        :timestamp => "Mar  1 15:43:35",
+        :hostname => "snack",
+        :message => "kernel: Kernel logging (proc) stopped."
+      },
+      "Jun 17 03:29:44 1.2.3.4 fancypants" => {
+        :pri => "13",
+        :timestamp => "Jun 17 03:29:44",
+        :hostname => "1.2.3.4",
+        :message => "fancypants"
+      },
+      "Dec 31 22:00:00 ffe0::1 something[12345]: hello world" => {
+        :pri => "13",
+        :timestamp => "Dec 31 22:00:00",
+        :hostname => "ffe0::1",
+        :message => "something[12345]: hello world",
+      },
+    }
+
+    event = LogPorter::Event.new
+    messages.each do |input, expect|
+      assert(parse_rfc3164(input, event), "Parse should return true on a valid message")
+
+      expect.each do |key, value|
+        actual = event.send(key.to_sym) # invoke 'event.message' or whatever 'key' is
+        assert_equal(value, actual, "Expected event.#{key} == #{value.inspect}, got #{actual.inspect}")
+      end
+    end
+  end # def test_valid_message_no_pri
+
+  def test_invalid_message
+    messages = [
+      " Mar  1 15:43:35 snack kernel: Kernel logging (proc) stopped.",
+      "<1234>Jun 17 03:29:44 1.2.3.4 fancypants",
+      "<123>Mon, Dec 31 22:00:00 ffe0::1 something[12345]: hello world",
+    ]
+
+    event = LogPorter::Event.new
+    messages.each do |input|
+      assert(parse_rfc3164(input, event) == false, "Parse should return false on a invalid message #{input.inspect}")
+    end
+  end # def test_invalid_message
+end # class TestSyslog3164Parser

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: logporter
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jordan Sissel
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-21 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: None yet.
+email: jordan@loggly.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/logporter/protocol/syslog3164.rb
+- lib/logporter/namespace.rb
+- lib/logporter/server.rb
+- lib/logporter/server/defaulthandler.rb
+- lib/logporter/server/connection.rb
+- lib/logporter/event.rb
+- test/syslog.rb
+has_rdoc: true
+homepage: https://github.com/loggly/logporter
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: logporter - a log server
+test_files: []
+