logstash-lite 0.2.20101118134500

data/lib/logstash/filters/grokdiscovery.rb

@@ -0,0 +1,60 @@
+require "logstash/filters/base"
+
+gem "jls-grok", ">=0.2.3071"
+require "grok" # rubygem 'jls-grok'
+
+class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
+  def initialize(config = {})
+    super
+
+    @discover_fields = {}
+  end # def initialize
+
+  def register
+    # TODO(sissel): Make patterns files come from the config
+    @config.each do |type, typeconfig|
+      @logger.debug("Registering type with grok: #{type}")
+      @grok = Grok.new
+      Dir.glob("patterns/*").each do |path|
+        @grok.add_patterns_from_file(path)
+      end
+      @discover_fields[type] = typeconfig
+      @logger.debug(["Enabling discovery", { :type => type, :fields => typeconfig }])
+      @logger.warn(@discover_fields)
+    end # @config.each
+  end # def register
+
+  def filter(event)
+    # parse it with grok
+    message = event.message
+    match = false
+
+    if event.type and @discover_fields.include?(event.type)
+      discover = @discover_fields[event.type] & event.fields.keys
+      discover.each do |field|
+        value = event.fields[field]
+        value = [value] if value.is_a?(String)
+
+        value.each do |v| 
+          pattern = @grok.discover(v)
+          @logger.warn("Trying #{v} => #{pattern}")
+          @grok.compile(pattern)
+          match = @grok.match(v)
+          if match
+            @logger.warn(["Match", match.captures])
+            event.fields.merge!(match.captures) do |key, oldval, newval|
+              @logger.warn(["Merging #{key}", oldval, newval])
+              oldval + newval # should both be arrays...
+            end
+          else
+            @logger.warn(["Discovery produced something not matchable?", { :input => v }])
+          end
+        end # value.each
+      end # discover.each
+    else
+      @logger.info("Unknown type for #{event.source} (type: #{event.type})")
+      @logger.debug(event.to_hash)
+    end
+    @logger.debug(["Event now: ", event.to_hash])
+  end # def filter
+end # class LogStash::Filters::Grokdiscovery

data/lib/logstash/inputs.rb

@@ -0,0 +1,18 @@
+
+require "logstash/namespace"
+require "uri"
+
+module LogStash::Inputs
+  def self.from_url(url, type, &block)
+    # Assume file paths if we start with "/"
+    url = "file://#{url}" if url.start_with?("/")
+
+    uri = URI.parse(url)
+    # TODO(sissel): Add error handling
+    # TODO(sissel): Allow plugin paths
+    klass = uri.scheme.capitalize
+    file = uri.scheme
+    require "logstash/inputs/#{file}"
+    LogStash::Inputs.const_get(klass).new(uri, type, &block)
+  end # def from_url
+end # module LogStash::Inputs

data/lib/logstash/inputs/amqp.rb

@@ -0,0 +1,48 @@
+require "logstash/inputs/base"
+require "amqp" # rubygem 'amqp'
+require "mq" # rubygem 'amqp'
+require "uuidtools" # rubygem 'uuidtools'
+
+class LogStash::Inputs::Amqp < LogStash::Inputs::Base
+  MQTYPES = [ "fanout", "queue", "topic" ]
+
+  def initialize(url, type, config={}, &block)
+    super
+
+    @mq = nil
+
+    # Handle path /<type>/<name>
+    unused, @mqtype, @name = @url.path.split("/", 3)
+    if @mqtype == nil or @name == nil
+      raise "amqp urls must have a path of /<type>/name where <type> is #{MQTYPES.join(", ")}"
+    end
+
+    if !MQTYPES.include?(@mqtype)
+      raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.JOIN(", ")}"
+    end
+  end
+
+  def register
+    @logger.info("Registering #{@url}")
+    @amqp = AMQP.connect(:host => @url.host)
+    @mq = MQ.new(@amqp)
+    @target = nil
+
+    @target = @mq.queue(UUIDTools::UUID.timestamp_create)
+    case @mqtype
+      when "fanout"
+        #@target.bind(MQ.fanout(@url.path, :durable => true))
+        @target.bind(@mq.fanout(@name))
+      when "direct"
+        @target.bind(@mq.direct(@name))
+      when "topic"
+        @target.bind(@mq.topic(@name))
+    end # case @mqtype
+
+    @target.subscribe(:ack => true) do |header, message|
+      event = LogStash::Event.from_json(message)
+      receive(event)
+      header.ack
+    end
+  end # def register
+end # class LogStash::Inputs::Amqp

data/lib/logstash/inputs/base.rb

@@ -0,0 +1,32 @@
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/logging"
+require "uri"
+
+class LogStash::Inputs::Base
+  def initialize(url, type, config={}, &block)
+    @logger = LogStash::Logger.new(STDERR)
+    @url = url
+    @url = URI.parse(url) if url.is_a? String
+    @config = config
+    @callback = block
+    @type = type
+    @tags = []
+  end
+
+  def register
+    raise "#{self.class}#register must be overidden"
+  end
+
+  def tag(newtag)
+    @tags << newtag
+  end
+
+  def receive(event)
+    @logger.debug(["Got event", { :url => @url, :event => event }])
+    # Only override the type if it doesn't have one
+    event.type = @type if !event.type 
+    event.tags |= @tags # set union
+    @callback.call(event)
+  end
+end # class LogStash::Inputs::Base

data/lib/logstash/inputs/file.rb

@@ -0,0 +1,47 @@
+require "logstash/inputs/base"
+require "eventmachine-tail"
+require "socket" # for Socket.gethostname
+
+class LogStash::Inputs::File < LogStash::Inputs::Base
+  def initialize(url, type, config={}, &block)
+    super
+
+    # Hack the hostname into the url.
+    # This works since file:// urls don't generally have a host in it.
+    @url.host = Socket.gethostname
+  end
+
+  def register
+    EventMachine::FileGlobWatchTail.new(@url.path, Reader, interval=60,
+                                        exclude=[], receiver=self)
+  end # def register
+
+  def receive(filetail, event)
+    url = @url.clone
+    url.path = filetail.path
+    @logger.debug(["original url", { :originalurl => @url, :newurl => url }])
+    event = LogStash::Event.new({
+      "@source" => url,
+      "@message" => event,
+      "@type" => @type,
+      "@tags" => @tags.clone,
+    })
+    @logger.debug(["Got event", event])
+    @callback.call(event)
+  end # def receive
+
+  class Reader < EventMachine::FileTail
+    def initialize(path, receiver)
+      super(path)
+      @receiver = receiver
+      @buffer = BufferedTokenizer.new  # From eventmachine
+    end
+
+    def receive_data(data)
+      # TODO(2.0): Support multiline log data
+      @buffer.extract(data).each do |line|
+        @receiver.receive(self, line)
+      end
+    end # def receive_data
+  end # class Reader
+end # class LogStash::Inputs::File

data/lib/logstash/inputs/syslog.rb

@@ -0,0 +1,123 @@
+require "logstash/inputs/base"
+require "eventmachine-tail"
+require "socket" # for Socket.gethostname
+require "date"
+require "logstash/time" # should really use the filters/date.rb bits
+
+
+class LogStash::Inputs::Syslog < LogStash::Inputs::Base
+  def register
+    if !@url.host or !@url.port
+      @logger.fatal("No host or port given in #{self.class}: #{@url}")
+      # TODO(sissel): Make this an actual exception class
+      raise "configuration error"
+    end
+
+    @logger.info("Starting tcp listener for #{@url}")
+    EventMachine::start_server(@url.host, @url.port, TCPInput, self, @logger)
+
+    @logger.info("Starting udp listener for #{@url}")
+    EventMachine::open_datagram_socket(@url.host, @url.port, UDPInput, self,
+                                       @logger)
+
+    # This comes from RFC3164, mostly.
+    @@syslog_re ||= \
+      /<([0-9]{1,3})>([A-z]{3} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) (\S+) (.*)/
+      #<priority       timestamp          Mmm dd hh:mm:ss             host  msg
+  end # def register
+
+  def receive(host, port, message)
+    url = @url.clone
+    url.host = host
+    url.port = port
+
+    # Do some syslog relay-like behavior.
+    # * Add syslog headers if there are none
+    event = LogStash::Event.new({
+      "@message" => message,
+      "@type" => @type,
+      "@tags" => @tags.clone,
+    })
+    syslog_relay(event, url)
+    @logger.debug(["Got event", event.class, event.to_hash])
+    @callback.call(event)
+  end # def receive
+
+  # Following RFC3164 where sane, we'll try to parse a received message
+  # as if you were relaying a syslog message to it.
+  # If the message cannot be recognized (see @@syslog_re), we'll
+  # treat it like the whole event.message is correct and try to fill
+  # the missing pieces (host, priority, etc)
+  def syslog_relay(event, url)
+    match = @@syslog_re.match(event.message)
+    if match
+      # match[1,2,3,4] = {pri, timestamp, hostname, message}
+      # Per RFC3164, priority = (facility * 8) + severity
+      #                       = (facility << 3) & (severity)
+      priority = match[1].to_i
+      severity = priority & 7   # 7 is 111 (3 bits)
+      facility = priority >> 3
+      event.fields["priority"] = priority
+      event.fields["severity"] = severity
+      event.fields["facility"] = facility
+     
+      # TODO(sissel): Use the date filter, somehow.
+      event.timestamp = LogStash::Time.to_iso8601(
+        DateTime.strptime(match[2], "%b %d %H:%M:%S"))
+
+      # At least the hostname is simple...
+      url.host = match[3]
+      url.port = nil
+      event.source = url
+
+      event.message = match[4]
+    else
+      @logger.info(["NOT SYSLOG", event.message])
+      url.host = Socket.gethostname if url.host == "127.0.0.1"
+
+      # RFC3164 says unknown messages get pri=13
+      priority = 13
+      severity = priority & 7   # 7 is 111 (3 bits)
+      facility = priority >> 3
+      event.fields["priority"] = 13
+      event.fields["severity"] = 5   # 13 & 7 == 5
+      event.fields["facility"] = 1   # 13 >> 3 == 1
+
+      # Don't need to modify the message, here.
+      # event.message = ...
+
+      event.source = url
+    end
+  end # def syslog_relay
+
+  class TCPInput < EventMachine::Connection
+    def initialize(receiver, logger)
+      @logger = logger
+      @receiver = receiver
+      @buffer = BufferedTokenizer.new  # From eventmachine
+    end # def initialize
+
+    # Messages over TCP may not be received all at once, chunk by newline.
+    def receive_data(data)
+      @buffer.extract(data).each do |line|
+        port, host = Socket.unpack_sockaddr_in(self.get_peername)
+        # Trim trailing newlines 
+        @receiver.receive(host, port, line.chomp)
+      end
+    end # def receive_data
+  end # class TCPInput
+
+  class UDPInput < EventMachine::Connection
+    def initialize(receiver, logger)
+      @logger = logger
+      @receiver = receiver
+    end # def initialize
+
+    # Every udp packet is a unique message.
+    def receive_data(data)
+      port, host = Socket.unpack_sockaddr_in(self.get_peername)
+      # Trim trailing newlines 
+      @receiver.receive(host, port, data.chomp)
+    end # def receive_data
+  end # class UDPInput
+end # class LogStash::Inputs::Tcp

data/lib/logstash/inputs/tcp.rb

@@ -0,0 +1,51 @@
+require "logstash/inputs/base"
+require "eventmachine-tail"
+require "socket" # for Socket.gethostname
+
+class LogStash::Inputs::Tcp < LogStash::Inputs::Base
+  def initialize(url, type, config={}, &block)
+    super
+  end
+
+  def register
+    if !@url.host or !@url.port
+      @logger.fatal("No host or port given in #{self.class}: #{@url}")
+      # TODO(sissel): Make this an actual exception class
+      raise "configuration error"
+    end
+
+    @logger.info("Starting tcp listener for #{@url}")
+    EventMachine::start_server(@url.host, @url.port, TCPInput, @url, self, @logger)
+  end
+
+  def receive(host, port, event)
+    url = @url.clone
+    url.host = host
+    url.port = port
+    @logger.debug(["original url", { :originalurl => @url, :newurl => url }])
+    event = LogStash::Event.new({
+      "@source" => url,
+      "@message" => event,
+      "@type" => @type,
+      "@tags" => @tags.clone,
+    })
+    @logger.debug(["Got event", event])
+    @callback.call(event)
+  end # def receive
+
+  class TCPInput < EventMachine::Connection
+    def initialize(url, receiver, logger)
+      @logger = logger
+      @receiver = receiver
+      @url = url;
+      @buffer = BufferedTokenizer.new  # From eventmachine
+    end # def initialize
+
+    def receive_data(data)
+      @buffer.extract(data).each do |line|
+        port, host = Socket.unpack_sockaddr_in(self.get_peername)
+        @receiver.receive(host, port, line)
+      end
+    end # def receive_data
+  end # class TCPInput
+end # class LogStash::Inputs::Tcp

data/lib/logstash/logging.rb

@@ -0,0 +1,82 @@
+require "logstash/namespace"
+require "logger"
+
+class LogStash::Logger < Logger
+  # Try to load awesome_print, if it fails, log it later
+  # but otherwise we should continue to operate as normal.
+  begin 
+    require "ap"
+    @@have_awesome_print = true
+  rescue LoadError => e
+    @@have_awesome_print = false
+    @@notify_awesome_print_load_failed = e
+  end
+
+  def initialize(*args)
+    super(*args)
+    @formatter = LogStash::Logger::Formatter.new
+
+    # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
+    self.level = $DEBUG ? Logger::DEBUG: Logger::INFO
+
+    # Conditional support for awesome_print
+    if !@@have_awesome_print && @@notify_awesome_print_load_failed
+      info [ "Failed: require 'ap' (aka awesome_print); some " \
+             "logging features may be disabled", 
+             @@notify_awesome_print_load_failed ]
+      @@notify_awesome_print_load_failed = nil
+    end
+
+    @formatter.progname = self.progname = File.basename($0)
+  end # def initialize
+
+  def level=(level)
+    super(level)
+    @formatter.level = level
+  end # def level=
+end # class LogStash::Logger
+
+# Implement a custom Logger::Formatter that uses awesome_inspect on non-strings.
+class LogStash::Logger::Formatter < Logger::Formatter
+  attr_accessor :level
+  attr_accessor :progname
+
+  def call(severity, timestamp, who, object)
+    # override progname to be the caller if the log level threshold is DEBUG
+    # We only do this if the logger level is DEBUG because inspecting the
+    # stack and doing extra string manipulation can have performance impacts
+    # under high logging rates.
+    if @level == Logger::DEBUG
+      # callstack inspection, include our caller
+      # turn this: "/usr/lib/ruby/1.8/irb/workspace.rb:52:in `irb_binding'"
+      # into this: ["/usr/lib/ruby/1.8/irb/workspace.rb", "52", "irb_binding"]
+      #
+      # caller[3] is actually who invoked the Logger#<type>
+      # This only works if you use the severity methods
+      path, line, method = caller[3].split(/(?::in `|:|')/)
+      # Trim RUBYLIB path from 'file' if we can
+      #whence = $:.select { |p| path.start_with?(p) }[0]
+      whence = $:.detect { |p| path.start_with?(p) }
+      if !whence
+        # We get here if the path is not in $:
+        file = path
+      else
+        file = path[whence.length + 1..-1]
+      end
+      who = "#{file}:#{line}##{method}"
+    end
+
+    # Log like normal if we got a string.
+    if object.is_a?(String)
+      super(severity, timestamp, who, object)
+    else
+      # If we logged an object, use .awesome_inspect (or just .inspect)
+      # to stringify it for higher sanity logging.
+      if object.respond_to?(:awesome_inspect)
+        super(severity, timestamp, who, object.awesome_inspect)
+      else
+        super(severity, timestamp, who, object.inspect)
+      end
+    end
+  end # def call
+end # class LogStash::Logger::Formatter