logstash-lib 1.3.2

data/lib/logstash/inputs/stomp.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require 'pp'
+
+class LogStash::Inputs::Stomp < LogStash::Inputs::Base
+  config_name "stomp"
+  milestone 2
+
+  default :codec, "plain"
+
+  # The address of the STOMP server.
+  config :host, :validate => :string, :default => "localhost", :required => true
+
+  # The port to connet to on your STOMP server.
+  config :port, :validate => :number, :default => 61613
+
+  # The username to authenticate with.
+  config :user, :validate => :string, :default => ""
+
+  # The password to authenticate with.
+  config :password, :validate => :password, :default => ""
+
+  # The destination to read events from.
+  #
+  # Example: "/topic/logstash"
+  config :destination, :validate => :string, :required => true
+
+  # The vhost to use
+  config :vhost, :validate => :string, :default => nil
+
+  # Enable debugging output?
+  config :debug, :validate => :boolean, :default => false
+
+  private
+  def connect
+    begin
+      @client.connect
+      @logger.debug("Connected to stomp server") if @client.connected?
+    rescue => e
+      @logger.debug("Failed to connect to stomp server, will retry", :exception => e, :backtrace => e.backtrace)
+      sleep 2
+      retry
+    end
+  end
+
+  public
+  def register
+    require "onstomp"
+    @client = OnStomp::Client.new("stomp://#{@host}:#{@port}", :login => @user, :passcode => @password.value)
+    @client.host = @vhost if @vhost
+    @stomp_url = "stomp://#{@user}:#{@password}@#{@host}:#{@port}/#{@destination}"
+
+    # Handle disconnects 
+    @client.on_connection_closed {
+      connect
+      subscription_handler # is required for re-subscribing to the destination
+    }
+    connect
+  end # def register
+
+  private
+  def subscription_handler
+    @client.subscribe(@destination) do |msg|
+      @codec.decode(msg.body) do |event|
+        decorate(event)
+        @output_queue << event
+      end
+    end
+    #In the event that there is only Stomp input plugin instances
+    #the process ends prematurely. The above code runs, and return
+    #the flow control to the 'run' method below. After that, the
+    #method "run_input" from agent.rb marks 'done' as 'true' and calls
+    #'finish' over the Stomp plugin instance.
+    #'Sleeping' the plugin leves the instance alive.
+    sleep
+  end
+
+  public
+  def run(output_queue)
+    @output_queue = output_queue 
+    subscription_handler
+  end # def run
+end # class LogStash::Inputs::Stomp

data/lib/logstash/inputs/syslog.rb

@@ -0,0 +1,237 @@
+# encoding: utf-8
+require "date"
+require "logstash/filters/grok"
+require "logstash/filters/date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+require "thread_safe"
+
+# Read syslog messages as events over the network.
+#
+# This input is a good choice if you already use syslog today.
+# It is also a good choice if you want to receive logs from
+# appliances and network devices where you cannot run your own
+# log collector.
+#
+# Of course, 'syslog' is a very muddy term. This input only supports RFC3164
+# syslog with some small modifications. The date format is allowed to be
+# RFC3164 style or ISO8601. Otherwise the rest of the RFC3164 must be obeyed.
+# If you do not use RFC3164, do not use this input.
+#
+# Note: this input will start listeners on both TCP and UDP
+class LogStash::Inputs::Syslog < LogStash::Inputs::Base
+  config_name "syslog"
+  milestone 1
+
+  default :codec, "plain"
+
+  # The address to listen on
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on. Remember that ports less than 1024 (privileged
+  # ports) may require root to use.
+  config :port, :validate => :number, :default => 514
+
+  # Use label parsing for severity and facility levels
+  config :use_labels, :validate => :boolean, :default => true
+
+  # Labels for facility levels
+  # This comes from RFC3164.
+  config :facility_labels, :validate => :array, :default => [ "kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7" ]
+
+  # Labels for severity levels
+  # This comes from RFC3164.
+  config :severity_labels, :validate => :array, :default => [ "Emergency" , "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug" ]
+
+  public
+  def initialize(params)
+    super
+    @shutdown_requested = false
+    BasicSocket.do_not_reverse_lookup = true
+  end # def initialize
+
+  public
+  def register
+    @grok_filter = LogStash::Filters::Grok.new(
+      "overwrite" => "message",
+      "match" => { "message" => "<%{POSINT:priority}>%{SYSLOGLINE}" },
+    )
+
+    @date_filter = LogStash::Filters::Date.new(
+      "match" => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"]
+    )
+
+    @grok_filter.register
+    @date_filter.register
+
+    @tcp_clients = ThreadSafe::Array.new
+  end # def register
+
+  public
+  def run(output_queue)
+    # udp server
+    udp_thr = Thread.new do
+      begin
+        udp_listener(output_queue)
+      rescue => e
+        break if @shutdown_requested
+        @logger.warn("syslog udp listener died",
+                     :address => "#{@host}:#{@port}", :exception => e,
+                     :backtrace => e.backtrace)
+        sleep(5)
+        retry
+      end # begin
+    end # Thread.new
+
+    # tcp server
+    tcp_thr = Thread.new do
+      begin
+        tcp_listener(output_queue)
+      rescue => e
+        break if @shutdown_requested
+        @logger.warn("syslog tcp listener died",
+                     :address => "#{@host}:#{@port}", :exception => e,
+                     :backtrace => e.backtrace)
+        sleep(5)
+        retry
+      end # begin
+    end # Thread.new
+
+    # If we exit and we're the only input, the agent will think no inputs
+    # are running and initiate a shutdown.
+    udp_thr.join
+    tcp_thr.join
+  end # def run
+
+  private
+  def udp_listener(output_queue)
+    @logger.info("Starting syslog udp listener", :address => "#{@host}:#{@port}")
+
+    if @udp
+      @udp.close
+    end
+
+    @udp = UDPSocket.new(Socket::AF_INET)
+    @udp.bind(@host, @port)
+
+    loop do
+      payload, client = @udp.recvfrom(9000)
+      # Ruby uri sucks, so don't use it.
+      @codec.decode(payload) do |event|
+        decorate(event)
+        event["host"] = client[3]
+        syslog_relay(event)
+        output_queue << event
+      end
+    end
+  ensure
+    close_udp
+  end # def udp_listener
+
+  private
+  def tcp_listener(output_queue)
+    @logger.info("Starting syslog tcp listener", :address => "#{@host}:#{@port}")
+    @tcp = TCPServer.new(@host, @port)
+    @tcp_clients = []
+
+    loop do
+      client = @tcp.accept
+      @tcp_clients << client
+      Thread.new(client) do |client|
+        ip, port = client.peeraddr[3], client.peeraddr[1]
+        @logger.info("new connection", :client => "#{ip}:#{port}")
+        LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}")
+        begin
+          client.each do |line|
+            @codec.decode(line) do |event|
+              decorate(event)
+              event["host"] = ip
+              syslog_relay(event)
+              output_queue << event
+            end
+          end
+        rescue Errno::ECONNRESET
+        ensure
+          @tcp_clients.delete(client)
+        end
+      end # Thread.new
+    end # loop do
+  ensure
+    close_tcp
+  end # def tcp_listener
+
+  public
+  def teardown
+    @shutdown_requested = true
+    close_udp
+    close_tcp
+    finished
+  end
+
+  private
+  def close_udp
+    if @udp
+      @udp.close_read rescue nil
+      @udp.close_write rescue nil
+    end
+    @udp = nil
+  end
+
+  private
+  def close_tcp
+    # If we somehow have this left open, close it.
+    @tcp_clients.each do |client|
+      client.close rescue nil
+    end
+    @tcp.close if @tcp rescue nil
+    @tcp = nil
+  end
+
+  # Following RFC3164 where sane, we'll try to parse a received message
+  # as if you were relaying a syslog message to it.
+  # If the message cannot be recognized (see @grok_filter), we'll
+  # treat it like the whole event["message"] is correct and try to fill
+  # the missing pieces (host, priority, etc)
+  public
+  def syslog_relay(event)
+    @grok_filter.filter(event)
+
+    if event["tags"].nil? || !event["tags"].include?("_grokparsefailure")
+      # Per RFC3164, priority = (facility * 8) + severity
+      #                       = (facility << 3) & (severity)
+      priority = event["priority"].first.to_i rescue 13
+      severity = priority & 7   # 7 is 111 (3 bits)
+      facility = priority >> 3
+      event["priority"] = priority
+      event["severity"] = severity
+      event["facility"] = facility
+
+      event["timestamp"] = event["timestamp8601"] if event.include?("timestamp8601")
+      @date_filter.filter(event)
+    else
+      @logger.info? && @logger.info("NOT SYSLOG", :message => event["message"])
+
+      # RFC3164 says unknown messages get pri=13
+      priority = 13
+      event["priority"] = 13
+      event["severity"] = 5   # 13 & 7 == 5
+      event["facility"] = 1   # 13 >> 3 == 1
+    end
+
+    # Apply severity and facility metadata if
+    # use_labels => true
+    if @use_labels
+      facility_number = event["facility"]
+      severity_number = event["severity"]
+
+      if @facility_labels[facility_number]
+        event["facility_label"] = @facility_labels[facility_number]
+      end
+
+      if @severity_labels[severity_number]
+        event["severity_label"] = @severity_labels[severity_number]
+      end
+    end
+  end # def syslog_relay
+end # class LogStash::Inputs::Syslog

data/lib/logstash/inputs/tcp.rb

@@ -0,0 +1,231 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/util/socket_peer"
+
+# Read events over a TCP socket.
+#
+# Like stdin and file inputs, each event is assumed to be one line of text.
+#
+# Can either accept connections from clients or connect to a server,
+# depending on `mode`.
+class LogStash::Inputs::Tcp < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+  config_name "tcp"
+  milestone 2
+
+  default :codec, "line"
+
+  # When mode is `server`, the address to listen on.
+  # When mode is `client`, the address to connect to.
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # When mode is `server`, the port to listen on.
+  # When mode is `client`, the port to connect to.
+  config :port, :validate => :number, :required => true
+
+  # The 'read' timeout in seconds. If a particular tcp connection is idle for
+  # more than this timeout period, we will assume it is dead and close it.
+  #
+  # If you never want to timeout, use -1.
+  config :data_timeout, :validate => :number, :default => -1
+
+  # Mode to operate in. `server` listens for client connections,
+  # `client` connects to a server.
+  config :mode, :validate => ["server", "client"], :default => "server"
+
+  # Enable ssl (must be set for other `ssl_` options to take effect)
+  config :ssl_enable, :validate => :boolean, :default => false
+
+  # Verify the identity of the other end of the ssl connection against the CA
+  # For input, sets the `@field.sslsubject` to that of the client certificate
+  config :ssl_verify, :validate => :boolean, :default => false
+
+  # ssl CA certificate, chainfile or CA path
+  # The system CA path is automatically included
+  config :ssl_cacert, :validate => :path
+
+  # ssl certificate
+  config :ssl_cert, :validate => :path
+
+  # ssl key
+  config :ssl_key, :validate => :path
+
+  # ssl key passphrase
+  config :ssl_key_passphrase, :validate => :password, :default => nil
+
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+
+  public
+  def register
+    require "socket"
+    require "timeout"
+    require "openssl"
+    if @ssl_enable
+      @ssl_context = OpenSSL::SSL::SSLContext.new
+      @ssl_context.cert = OpenSSL::X509::Certificate.new(File.read(@ssl_cert))
+      @ssl_context.key = OpenSSL::PKey::RSA.new(File.read(@ssl_key),@ssl_key_passphrase)
+      if @ssl_verify
+        @cert_store = OpenSSL::X509::Store.new
+        # Load the system default certificate path to the store
+        @cert_store.set_default_paths
+        if File.directory?(@ssl_cacert)
+          @cert_store.add_path(@ssl_cacert)
+        else
+          @cert_store.add_file(@ssl_cacert)
+        end
+        @ssl_context.cert_store = @cert_store
+        @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+      end
+    end # @ssl_enable
+
+    if server?
+      @logger.info("Starting tcp input listener", :address => "#{@host}:#{@port}")
+      begin
+        @server_socket = TCPServer.new(@host, @port)
+      rescue Errno::EADDRINUSE
+        @logger.error("Could not start TCP server: Address in use",
+                      :host => @host, :port => @port)
+        raise
+      end
+      if @ssl_enable
+        @server_socket = OpenSSL::SSL::SSLServer.new(@server_socket, @ssl_context)
+      end # @ssl_enable
+    end
+  end # def register
+
+  private
+  def handle_socket(socket, client_address, output_queue, codec)
+    while true
+      buf = nil
+      # NOTE(petef): the timeout only hits after the line is read
+      # or socket dies
+      # TODO(sissel): Why do we have a timeout here? What's the point?
+      if @data_timeout == -1
+        buf = read(socket)
+      else
+        Timeout::timeout(@data_timeout) do
+          buf = read(socket)
+        end
+      end
+      codec.decode(buf) do |event|
+        decorate(event)
+        event["host"] = client_address
+        event["sslsubject"] = socket.peer_cert.subject if @ssl_enable && @ssl_verify
+        output_queue << event
+      end
+    end # loop do
+  rescue => e
+    codec.respond_to?(:flush) && codec.flush do |event|
+      decorate(event)
+      event["host"] = client_address
+      event["sslsubject"] = socket.peer_cert.subject if @ssl_enable && @ssl_verify
+      output_queue << event
+    end
+
+    @logger.debug("An error occurred. Closing connection",
+                  :client => socket.peer, :exception => e)
+  ensure
+    begin
+      socket.close
+    rescue IOError
+      #pass
+    end # begin
+  end
+
+  private
+  def server?
+    @mode == "server"
+  end # def server?
+
+  private
+  def read(socket)
+    return socket.sysread(16384)
+  end # def readline
+
+  public
+  def run(output_queue)
+    if server?
+      run_server(output_queue)
+    else
+      run_client(output_queue)
+    end
+  end # def run
+
+  def run_server(output_queue)
+    @thread = Thread.current
+    @client_threads = []
+    loop do
+      # Start a new thread for each connection.
+      begin
+        @client_threads << Thread.start(@server_socket.accept) do |s|
+          # TODO(sissel): put this block in its own method.
+
+          # monkeypatch a 'peer' method onto the socket.
+          s.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+          @logger.debug("Accepted connection", :client => s.peer,
+                        :server => "#{@host}:#{@port}")
+          begin
+            handle_socket(s, s.peer, output_queue, @codec.clone)
+          rescue Interrupted
+            s.close rescue nil
+          end
+        end # Thread.start
+      rescue OpenSSL::SSL::SSLError => ssle
+        # NOTE(mrichar1): This doesn't return a useful error message for some reason
+        @logger.error("SSL Error", :exception => ssle,
+                      :backtrace => ssle.backtrace)
+      rescue IOError, LogStash::ShutdownSignal
+        if @interrupted
+          # Intended shutdown, get out of the loop
+          @server_socket.close
+          @client_threads.each do |thread|
+            thread.raise(LogStash::ShutdownSignal)
+          end
+          break
+        else
+          # Else it was a genuine IOError caused by something else, so propagate it up..
+          raise
+        end
+      end
+    end # loop
+  rescue LogStash::ShutdownSignal
+    # nothing to do
+  ensure
+    @server_socket.close
+  end # def run_server
+
+  def run_client(output_queue) 
+    @thread = Thread.current
+    while true
+      client_socket = TCPSocket.new(@host, @port)
+      if @ssl_enable
+        client_socket = OpenSSL::SSL::SSLSocket.new(client_socket, @ssl_context)
+        begin
+          client_socket.connect
+        rescue OpenSSL::SSL::SSLError => ssle
+          @logger.error("SSL Error", :exception => ssle,
+                        :backtrace => ssle.backtrace)
+          # NOTE(mrichar1): Hack to prevent hammering peer
+          sleep(5)
+          next
+        end
+      end
+      client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+      @logger.debug("Opened connection", :client => "#{client_socket.peer}")
+      handle_socket(client_socket, client_socket.peer, output_queue, @codec.clone)
+    end # loop
+  ensure
+    client_socket.close
+  end # def run
+
+  public
+  def teardown
+    if server?
+      @interrupted = true
+      @thread.raise(LogStash::ShutdownSignal)
+    end
+  end # def teardown
+end # class LogStash::Inputs::Tcp