logstash-lib 1.3.2

data/lib/logstash/inputs/lumberjack.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# Receive events using the lumberjack protocol.
+#
+# This is mainly to receive events shipped  with lumberjack,
+# <http://github.com/jordansissel/lumberjack>
+class LogStash::Inputs::Lumberjack < LogStash::Inputs::Base
+
+  config_name "lumberjack"
+  milestone 1
+
+  default :codec, "plain"
+
+  # the address to listen on.
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # the port to listen on.
+  config :port, :validate => :number, :required => true
+
+  # ssl certificate to use
+  config :ssl_certificate, :validate => :path, :required => true
+
+  # ssl key to use
+  config :ssl_key, :validate => :path, :required => true
+
+  # ssl key passphrase to use
+  config :ssl_key_passphrase, :validate => :password
+
+  # TODO(sissel): Add CA to authenticate clients with.
+
+  public
+  def register
+    require "lumberjack/server"
+
+    @logger.info("Starting lumberjack input listener", :address => "#{@host}:#{@port}")
+    @lumberjack = Lumberjack::Server.new(:address => @host, :port => @port,
+      :ssl_certificate => @ssl_certificate, :ssl_key => @ssl_key,
+      :ssl_key_passphrase => @ssl_key_passphrase)
+  end # def register
+
+  public
+  def run(output_queue)
+    @lumberjack.run do |l|
+      @codec.decode(l.delete("line")) do |event|
+        decorate(event)
+        l.each { |k,v| event[k] = v; v.force_encoding("UTF-8") }
+        output_queue << event
+      end
+    end
+  end # def run
+end # class LogStash::Inputs::Lumberjack

data/lib/logstash/inputs/pipe.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+# Stream events from a long running command pipe.
+#
+# By default, each event is assumed to be one line. If you
+# want to join lines, you'll want to use the multiline filter.
+#
+class LogStash::Inputs::Pipe < LogStash::Inputs::Base
+  config_name "pipe"
+  milestone 1
+
+  # TODO(sissel): This should switch to use the 'line' codec by default
+  # once we switch away from doing 'readline'
+  default :codec, "plain"
+
+  # Command to run and read events from, one line at a time.
+  #
+  # Example:
+  #
+  #    command => "echo hello world"
+  config :command, :validate => :string, :required => true
+
+  public
+  def register
+    @logger.info("Registering pipe input", :command => @command)
+  end # def register
+
+  public
+  def run(queue)
+    loop do
+      begin
+        @pipe = IO.popen(@command, mode="r")
+        hostname = Socket.gethostname
+
+        @pipe.each do |line|
+          line = line.chomp
+          source = "pipe://#{hostname}/#{@command}"
+          @logger.debug? && @logger.debug("Received line", :command => @command, :line => line)
+          @codec.decode(line) do |event|
+            event["host"] = hostname
+            event["command"] = @command
+            decorate(event)
+            queue << event
+          end
+        end
+      rescue Exception => e
+        @logger.error("Exception while running command", :e => e, :backtrace => e.backtrace)
+      end
+
+      # Keep running the command forever.
+      sleep(10)
+    end
+  end # def run
+end # class LogStash::Inputs::Pipe

data/lib/logstash/inputs/rabbitmq.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+
+# Pull events from a RabbitMQ exchange.
+#
+# The default settings will create an entirely transient queue and listen for all messages by default.
+# If you need durability or any other advanced settings, please set the appropriate options
+#
+# This has been tested with Bunny 0.9.x, which supports RabbitMQ 2.x and 3.x. You can
+# find links to both here:
+#
+# * RabbitMQ - <http://www.rabbitmq.com/>
+# * March Hare: <http://rubymarchhare.info>
+# * Bunny - <https://github.com/ruby-amqp/bunny>
+class LogStash::Inputs::RabbitMQ < LogStash::Inputs::Threadable
+
+  config_name "rabbitmq"
+  milestone 1
+
+  #
+  # Connection
+  #
+
+  # RabbitMQ server address
+  config :host, :validate => :string, :required => true
+
+  # RabbitMQ port to connect on
+  config :port, :validate => :number, :default => 5672
+
+  # RabbitMQ username
+  config :user, :validate => :string, :default => "guest"
+
+  # RabbitMQ password
+  config :password, :validate => :password, :default => "guest"
+
+  # The vhost to use. If you don't know what this is, leave the default.
+  config :vhost, :validate => :string, :default => "/"
+
+  # Enable or disable SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # Validate SSL certificate
+  config :verify_ssl, :validate => :boolean, :default => false
+
+  # Enable or disable logging
+  config :debug, :validate => :boolean, :default => false
+
+
+
+  #
+  # Queue & Consumer
+  #
+
+  # The name of the queue Logstash will consume events from.
+  config :queue, :validate => :string, :default => ""
+
+  # Is this queue durable? (aka; Should it survive a broker restart?)
+  config :durable, :validate => :boolean, :default => false
+
+  # Should the queue be deleted on the broker when the last consumer
+  # disconnects? Set this option to 'false' if you want the queue to remain
+  # on the broker, queueing up messages until a consumer comes along to
+  # consume them.
+  config :auto_delete, :validate => :boolean, :default => true
+
+  # Is the queue exclusive? (aka: Will other clients connect to this named queue?)
+  config :exclusive, :validate => :boolean, :default => true
+
+  # Extra queue arguments as an array.
+  # To make a RabbitMQ queue mirrored, use: {"x-ha-policy" => "all"}
+  config :arguments, :validate => :array, :default => {}
+
+  # Prefetch count. Number of messages to prefetch
+  config :prefetch_count, :validate => :number, :default => 256
+
+  # Enable message acknowledgement
+  config :ack, :validate => :boolean, :default => true
+
+  # Passive queue creation? Useful for checking queue existance without modifying server state
+  config :passive, :validate => :boolean, :default => false
+
+
+
+  #
+  # (Optional) Exchange binding
+  #
+
+  # Optional.
+  #
+  # The name of the exchange to bind the queue to.
+  config :exchange, :validate => :string
+
+  # Optional.
+  #
+  # The routing key to use when binding a queue to the exchange.
+  # This is only relevant for direct or topic exchanges.
+  #
+  # * Routing keys are ignored on fanout exchanges.
+  # * Wildcards are not valid on direct exchanges.
+  config :key, :validate => :string, :default => "logstash"
+
+
+  def initialize(params)
+    params["codec"] = "json" if !params["codec"]
+
+    super
+  end
+
+  # Use March Hare on JRuby to avoid IO#select CPU spikes
+  # (see github.com/ruby-amqp/bunny/issues/95).
+  #
+  # On MRI, use Bunny.
+  #
+  # See http://rubybunny.info and http://rubymarchhare.info
+  # for the docs.
+  if RUBY_ENGINE == "jruby"
+    require "logstash/inputs/rabbitmq/march_hare"
+
+    include MarchHareImpl
+  else
+    require "logstash/inputs/rabbitmq/bunny"
+
+    include BunnyImpl
+  end
+end # class LogStash::Inputs::RabbitMQ

data/lib/logstash/inputs/rabbitmq/bunny.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+class LogStash::Inputs::RabbitMQ
+  module BunnyImpl
+    def register
+      require "bunny"
+
+      @vhost       ||= Bunny::DEFAULT_HOST
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= AMQ::Protocol::DEFAULT_PORT
+      @routing_key ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :automatically_recover => false
+      }
+      @settings[:user]      = @user || Bunny::DEFAULT_USER
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                Bunny::DEFAULT_PASSWORD
+                              end
+
+      @settings[:log_level] = if @debug
+                                :debug
+                              else
+                                :error
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      @settings[:verify_ssl] = @verify_ssl if @verify_ssl
+
+      proto                  = if @ssl
+                                 "amqp"
+                               else
+                                 "amqps"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      @logger.info("Registering input #{@connection_url}")
+    end
+
+    def run(output_queue)
+      @output_queue = output_queue
+
+      begin
+        setup
+        consume
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        # Because we manually reconnect instead of letting Bunny
+        # handle failures,
+        # make sure we don't leave any consumer work pool
+        # threads behind. MK.
+        @ch.maybe_kill_consumer_work_pool!
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
+
+        sleep n
+        retry
+      end
+    end
+
+    def teardown
+      @consumer.cancel
+      @q.delete unless @durable
+
+      @ch.close   if @ch && @ch.open?
+      @conn.close if @conn && @conn.open?
+
+      finished
+    end
+
+    def setup
+      @conn = Bunny.new(@settings)
+
+      @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+      return if terminating?
+      @conn.start
+
+      @ch = @conn.create_channel.tap do |ch|
+        ch.prefetch(@prefetch_count)
+      end
+      @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+
+      @arguments_hash = Hash[*@arguments]
+
+      @q = @ch.queue(@queue,
+                     :durable     => @durable,
+                     :auto_delete => @auto_delete,
+                     :exclusive   => @exclusive,
+                     :arguments   => @arguments)
+
+      # exchange binding is optional for the input
+      if @exchange
+        @q.bind(@exchange, :routing_key => @key)
+      end
+    end
+
+    def consume
+      @logger.info("Will consume events from queue #{@q.name}")
+
+      # we both need to block the caller in Bunny::Queue#subscribe and have
+      # a reference to the consumer so that we can cancel it, so
+      # a consumer manually. MK.
+      @consumer = Bunny::Consumer.new(@ch, @q)
+      @q.subscribe(:manual_ack => @ack, :block => true) do |delivery_info, properties, data|
+        @codec.decode(data) do |event|
+          decorate(event)
+          @output_queue << event
+        end
+
+        @ch.acknowledge(delivery_info.delivery_tag) if @ack
+      end
+    end
+  end # BunnyImpl
+end

data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb

@@ -0,0 +1 @@
+require "logstash/inputs/rabbitmq/march_hare"

data/lib/logstash/inputs/rabbitmq/march_hare.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+class LogStash::Inputs::RabbitMQ
+  # MarchHare-based implementation for JRuby
+  module MarchHareImpl
+    def register
+      require "hot_bunnies"
+      require "java"
+
+      @vhost       ||= "127.0.0.1"
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= 5672
+      @key         ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :user  => @user,
+        :automatic_recovery => false
+      }
+      @settings[:pass]      = @password.value if @password
+      @settings[:tls]       = @ssl if @ssl
+
+      proto                 = if @ssl
+                                "amqp"
+                              else
+                                "amqps"
+                              end
+      @connection_url       = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      @logger.info("Registering input #{@connection_url}")
+    end
+
+    def run(output_queue)
+      @output_queue          = output_queue
+      @break_out_of_the_loop = java.util.concurrent.atomic.AtomicBoolean.new(false)
+
+      # MarchHare does not raise exceptions when connection goes down with a blocking
+      # consumer running (it uses callbacks, as the RabbitMQ Java client does).
+      #
+      # However, MarchHare::Channel will make sure to unblock all blocking consumers
+      # on any internal shutdown, so #consume will return and another loop iteration
+      # will run.
+      #
+      # This is very similar to how the Bunny implementation works and is sufficient
+      # for our needs: it recovers successfully after RabbitMQ is kill -9ed, the
+      # network device is shut down, etc. MK.
+      until @break_out_of_the_loop.get do
+        begin
+          setup
+          consume
+        rescue MarchHare::Exception, java.lang.Throwable, com.rabbitmq.client.AlreadyClosedException => e
+          n = 10
+          @logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
+
+          sleep n
+          retry
+        rescue LogStash::ShutdownSignal => ss
+          shutdown_consumer
+        end
+
+        n = 10
+        @logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
+      end
+    end
+
+    def teardown
+      shutdown_consumer
+      @q.delete unless @durable
+
+      @ch.close         if @ch && @ch.open?
+      @connection.close if @connection && @connection.open?
+
+      finished
+    end
+
+    #
+    # Implementation
+    #
+
+    protected
+
+    def setup
+      return if terminating?
+
+      @conn = MarchHare.connect(@settings)
+      @logger.info("Connected to RabbitMQ #{@connection_url}")
+
+      @ch          = @conn.create_channel.tap do |ch|
+        ch.prefetch = @prefetch_count
+      end
+
+      @arguments_hash = Hash[*@arguments]
+
+      @q = @ch.queue(@queue,
+        :durable     => @durable,
+        :auto_delete => @auto_delete,
+        :exclusive   => @exclusive,
+        :arguments   => @arguments)
+
+      # exchange binding is optional for the input
+      if @exchange
+        @q.bind(@exchange, :routing_key => @key)
+      end
+    end
+
+    def consume
+      return if terminating?
+
+      # we manually build a consumer here to be able to keep a reference to it
+      # in an @ivar even though we use a blocking version of HB::Queue#subscribe
+      @consumer = @q.build_consumer(:block => true) do |metadata, data|
+        @codec.decode(data) do |event|
+          decorate(event)
+          @output_queue << event if event
+        end
+        @ch.ack(metadata.delivery_tag) if @ack
+      end
+      @q.subscribe_with(@consumer, :manual_ack => @ack, :block => true)
+    end
+
+    def shutdown_consumer
+      @break_out_of_the_loop.set(true)
+
+      @consumer.cancel
+      @consumer.gracefully_shut_down
+    end
+  end # MarchHareImpl
+end