logstash-lib 1.3.2

data/lib/logstash/inputs/graphite.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/inputs/tcp"
+require "logstash/namespace"
+
+# Receive graphite metrics. This plugin understands the text-based graphite
+# carbon protocol. Both 'N' and specific-timestamp forms are supported, example:
+#
+#     mysql.slow_query.count 204 N
+#     haproxy.live_backends 7 1364608909
+#
+# 'N' means 'now' for a timestamp. This plugin also supports having the time
+# specified in the metric payload:
+#
+# For every metric received from a client, a single event will be emitted with
+# the metric name as the field (like 'mysql.slow_query.count') and the metric
+# value as the field's value.
+class LogStash::Inputs::Graphite < LogStash::Inputs::Tcp
+  config_name "graphite"
+  milestone 1
+
+  ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
+
+  public
+  def run(output_queue)
+    @queue = output_queue
+    super(self)
+  end
+
+  # This is a silly hack to make the superclass (Tcp) give us a finished event
+  # so that we can parse it accordingly.
+  def <<(event)
+    name, value, time = event["message"].split(" ")
+    event[name] = value.to_f
+
+    if time != "N"
+      event["@timestamp"] = Time.at(time.to_i).gmtime
+    end
+
+    @queue << event
+  end
+end # class LogStash::Inputs::Graphite

data/lib/logstash/inputs/heroku.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# Stream events from a heroku app's logs.
+#
+# This will read events in a manner similar to how the `heroku logs -t` command
+# fetches logs.
+#
+# Recommended filters:
+#
+#     filter {
+#       grok {
+#         pattern => "^%{TIMESTAMP_ISO8601:timestamp} %{WORD:component}\[%{WORD:process}(?:\.%{INT:instance:int})?\]: %{DATA:message}$"
+#       }
+#       date { timestamp => ISO8601 }
+#     }
+class LogStash::Inputs::Heroku < LogStash::Inputs::Base
+  config_name "heroku"
+  milestone 1
+
+  default :codec, "plain"
+
+  # The name of your heroku application. This is usually the first part of the 
+  # the domain name 'my-app-name.herokuapp.com'
+  config :app, :validate => :string, :required => true
+
+  public
+  def register
+    require "heroku"
+    require "logstash/util/buftok"
+  end # def register
+
+  public
+  def run(queue)
+    client = Heroku::Client.new(Heroku::Auth.user, Heroku::Auth.password)
+
+    # The 'Herok::Client#read_logs' method emits chunks of text not bounded
+    # by event barriers like newlines.
+    # tail=1 means to follow logs
+    # I *think* setting num=1 means we only get 1 historical event. Setting
+    # this to 0 makes it fetch *all* events, not what I want.
+    client.read_logs(@app, ["tail=1", "num=1"]) do |chunk|
+      @codec.decode(chunk) do |event|
+        decorate(event)
+        event["app"] = @app
+        queue << event
+      end
+    end
+  end # def run
+end # class LogStash::Inputs::Heroku

data/lib/logstash/inputs/imap.rb

@@ -0,0 +1,136 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require "socket" # for Socket.gethostname
+
+# Read mail from IMAP servers
+#
+# Periodically scans INBOX and moves any read messages
+# to the trash.
+class LogStash::Inputs::IMAP < LogStash::Inputs::Base
+  config_name "imap"
+  milestone 1
+  ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
+
+  config :host, :validate => :string, :required => true
+  config :port, :validate => :number
+
+  config :user, :validate => :string, :required => true
+  config :password, :validate => :password, :required => true
+  config :secure, :validate => :boolean, :default => true
+
+  config :fetch_count, :validate => :number, :default => 50
+  config :lowercase_headers, :validate => :boolean, :default => true
+  config :check_interval, :validate => :number, :default => 300
+  config :delete, :validate => :boolean, :default => false
+
+  # For multipart messages, use the first part that has this
+  # content-type as the event message.
+  config :content_type, :validate => :string, :default => "text/plain"
+
+  public
+  def register
+    require "net/imap" # in stdlib
+    require "mail" # gem 'mail'
+
+    if @port.nil?
+      if @secure
+        @port = 993
+      else
+        @port = 143
+      end
+    end
+
+    @content_type_re = Regexp.new("^" + @content_type)
+  end # def register
+
+  def connect
+    imap = Net::IMAP.new(@host, :port => @port, :ssl => @secure)
+    imap.login(@user, @password.value)
+    return imap
+  end
+
+  def run(queue)
+    Stud.interval(@check_interval) do
+      check_mail(queue)
+    end
+  end
+
+  def check_mail(queue)
+    # TODO(sissel): handle exceptions happening during runtime:
+    # EOFError, OpenSSL::SSL::SSLError
+    imap = connect
+    imap.select("INBOX")
+    ids = imap.search("NOT SEEN")
+
+    ids.each_slice(@fetch_count) do |id_set|
+      items = imap.fetch(id_set, "RFC822")
+      items.each do |item|
+        next unless item.attr.has_key?("RFC822")
+        mail = Mail.read_from_string(item.attr["RFC822"])
+        queue << parse_mail(mail)
+      end
+
+      imap.store(id_set, '+FLAGS', @delete ? :Deleted : :Seen)
+    end
+
+    imap.close
+    imap.disconnect
+  end # def run
+
+  def parse_mail(mail)
+    # TODO(sissel): What should a multipart message look like as an event?
+    # For now, just take the plain-text part and set it as the message.
+    if mail.parts.count == 0
+      # No multipart message, just use the body as the event text
+      message = mail.body.decoded
+    else
+      # Multipart message; use the first text/plain part we find
+      part = mail.parts.find { |p| p.content_type.match @content_type_re } || mail.parts.first
+      message = part.decoded
+    end
+
+    event = LogStash::Event.new("message" => message)
+
+    # Use the 'Date' field as the timestamp
+    event["@timestamp"] = mail.date.to_time.gmtime
+
+    # Add fields: Add message.header_fields { |h| h.name=> h.value }
+    mail.header_fields.each do |header|
+      if @lowercase_headers
+        # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in
+        # String form
+        name = header.name.to_s.downcase
+      else
+        name = header.name.to_s
+      end
+      # Call .decoded on the header in case it's in encoded-word form.
+      # Details at:
+      #   https://github.com/mikel/mail/blob/master/README.md#encodings
+      #   http://tools.ietf.org/html/rfc2047#section-2
+      value = header.decoded
+
+      # Assume we already processed the 'date' above.
+      next if name == "Date"
+
+      case event[name]
+        # promote string to array if a header appears multiple times
+        # (like 'received')
+        when String; event[name] = [event[name], value]
+        when Array; event[name].is_a?(Array)
+        when nil; event[name] = value
+      end
+    end # mail.header_fields.each
+
+    decorate(event)
+
+    return event
+  end # def handle
+
+  public
+  def teardown
+    $stdin.close
+    finished
+  end # def teardown
+end # class LogStash::Inputs::IMAP

data/lib/logstash/inputs/irc.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "thread"
+
+# Read events from an IRC Server.
+#
+class LogStash::Inputs::Irc < LogStash::Inputs::Base
+
+  config_name "irc"
+  milestone 1
+
+  default :codec, "plain"
+
+  # Host of the IRC Server to connect to.
+  config :host, :validate => :string, :required => true
+
+  # Port for the IRC Server
+  config :port, :validate => :number, :default => 6667
+
+  # Set this to true to enable SSL.
+  config :secure, :validate => :boolean, :default => false
+
+  # IRC Nickname
+  config :nick, :validate => :string, :default => "logstash"
+
+  # IRC Username
+  config :user, :validate => :string, :default => "logstash"
+
+  # IRC Real name
+  config :real, :validate => :string, :default => "logstash"
+
+  # IRC Server password
+  config :password, :validate => :password
+
+  # Channels to join and read messages from.
+  #
+  # These should be full channel names including the '#' symbol, such as
+  # "#logstash".
+  config :channels, :validate => :array, :required => true
+
+  public
+  def register
+    require "cinch"
+    @irc_queue = Queue.new
+    @logger.info("Connecting to irc server", :host => @host, :port => @port, :nick => @nick, :channels => @channels)
+
+    @bot = Cinch::Bot.new
+    @bot.loggers.clear
+    @bot.configure do |c|
+      c.server = @host
+      c.port = @port
+      c.nick = @nick
+      c.user = @user
+      c.realname = @real
+      c.channels = @channels
+      c.password = @password.value rescue nil
+      c.ssl.use = @secure
+    end
+    queue = @irc_queue
+    @bot.on :channel  do |m|
+      queue << m
+    end
+  end # def register
+
+  public
+  def run(output_queue)
+    Thread.new(@bot) do |bot|
+      bot.start
+    end
+    loop do
+      msg = @irc_queue.pop
+      if msg.user
+        @codec.decode(msg.message) do |event|
+          decorate(event)
+          event["channel"] = msg.channel.to_s
+          event["nick"] = msg.user.nick
+          event["server"] = "#{@host}:#{@port}"
+          output_queue << event
+        end
+      end
+    end
+  end # def run
+end # class LogStash::Inputs::Irc

data/lib/logstash/inputs/log4j.rb

@@ -0,0 +1,136 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/util/socket_peer"
+require "socket"
+require "timeout"
+
+# Read events over a TCP socket from Log4j SocketAppender.
+#
+# Can either accept connections from clients or connect to a server,
+# depending on `mode`. Depending on mode, you need a matching SocketAppender or SocketHubAppender on the remote side
+class LogStash::Inputs::Log4j < LogStash::Inputs::Base
+
+  config_name "log4j"
+  milestone 1
+
+  # When mode is `server`, the address to listen on.
+  # When mode is `client`, the address to connect to.
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # When mode is `server`, the port to listen on.
+  # When mode is `client`, the port to connect to.
+  config :port, :validate => :number, :required => true
+
+  # Read timeout in seconds. If a particular tcp connection is
+  # idle for more than this timeout period, we will assume
+  # it is dead and close it.
+  # If you never want to timeout, use -1.
+  config :data_timeout, :validate => :number, :default => 5
+
+  # Mode to operate in. `server` listens for client connections,
+  # `client` connects to a server.
+  config :mode, :validate => ["server", "client"], :default => "server"
+
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    require "java"
+    require "jruby/serialization"
+
+    if __FILE__ !~ /^(jar:)?file:\/\//
+      if File.exists?("vendor/jar/elasticsearch-0.90.3/lib/log4j-1.2.17.jar")
+        require "vendor/jar/elasticsearch-0.90.3/lib/log4j-1.2.17.jar"
+      end
+    end
+
+    if server?
+      @logger.info("Starting Log4j input listener", :address => "#{@host}:#{@port}")
+      @server_socket = TCPServer.new(@host, @port)
+    end
+    @logger.info("Log4j input")
+  end # def register
+
+  private
+  def handle_socket(socket, output_queue)
+    begin
+      # JRubyObjectInputStream uses JRuby class path to find the class to de-serialize to
+      ois = JRubyObjectInputStream.new(java.io.BufferedInputStream.new(socket.to_inputstream))
+      loop do
+        # NOTE: event_raw is org.apache.log4j.spi.LoggingEvent
+        log4j_obj = ois.readObject
+        event = LogStash::Event.new("message" => log4j_obj.getRenderedMessage)
+        decorate(event)
+        event["host"] = socket.peer
+        event["path"] = log4j_obj.getLoggerName
+        event["priority"] = log4j_obj.getLevel.toString
+        event["logger_name"] = log4j_obj.getLoggerName
+        event["thread"] = log4j_obj.getThreadName
+        event["class"] = log4j_obj.getLocationInformation.getClassName
+        event["file"] = log4j_obj.getLocationInformation.getFileName + ":" + log4j_obj.getLocationInformation.getLineNumber
+        event["method"] = log4j_obj.getLocationInformation.getMethodName
+        event["NDC"] = log4j_obj.getNDC if log4j_obj.getNDC
+        event["stack_trace"] = log4j_obj.getThrowableStrRep.to_a.join("\n") if log4j_obj.getThrowableInformation
+        
+        # Add the MDC context properties to '@fields'
+        if log4j_obj.getProperties
+          log4j_obj.getPropertyKeySet.each do |key|
+            event[key] = log4j_obj.getProperty(key)
+          end  
+        end  
+
+        output_queue << event
+      end # loop do
+    rescue => e
+      @logger.debug("Closing connection", :client => socket.peer,
+                    :exception => e)
+    rescue Timeout::Error
+      @logger.debug("Closing connection after read timeout",
+                    :client => socket.peer)
+    end # begin
+  ensure
+    begin
+      socket.close
+    rescue IOError
+      pass
+    end # begin
+  end
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+  private
+  def readline(socket)
+    line = socket.readline
+  end # def readline
+
+  public
+  def run(output_queue)
+    if server?
+      loop do
+        # Start a new thread for each connection.
+        Thread.start(@server_socket.accept) do |s|
+          # TODO(sissel): put this block in its own method.
+
+          # monkeypatch a 'peer' method onto the socket.
+          s.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+          @logger.debug("Accepted connection", :client => s.peer,
+                        :server => "#{@host}:#{@port}")
+          handle_socket(s, output_queue)
+        end # Thread.start
+      end # loop
+    else
+      loop do
+        client_socket = TCPSocket.new(@host, @port)
+        client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+        @logger.debug("Opened connection", :client => "#{client_socket.peer}")
+        handle_socket(client_socket, output_queue)
+      end # loop
+    end
+  end # def run
+end # class LogStash::Inputs::Log4j