logstash-input-rabbitmq 0.1.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c3040583aacf0c457c56df3656d0b5a2e2739a63
+  data.tar.gz: 0305b94426b028b7e426e80cc13f69501455df25
+SHA512:
+  metadata.gz: d109d797d6b337b5fc301b606a010fc7ab1d103a4330f57c0cec680c298a1379b145caa4952e5fa8d0c5c451de252db194f94aa543da15ef1a6577674983c756
+  data.tar.gz: ea4a54f7c54af4684062e214231e683783633b81ce71fcccadca5b804f425bc56c294b98a80743904ef1e767ba791a09a1b1752f98f9f0dac3672a148b720a1f

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/inputs/rabbitmq.rb

@@ -0,0 +1,128 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+
+# Pull events from a RabbitMQ exchange.
+#
+# The default settings will create an entirely transient queue and listen for all messages by default.
+# If you need durability or any other advanced settings, please set the appropriate options
+#
+# This has been tested with Bunny 0.9.x, which supports RabbitMQ 2.x and 3.x. You can
+# find links to both here:
+#
+# * RabbitMQ - <http://www.rabbitmq.com/>
+# * March Hare: <http://rubymarchhare.info>
+# * Bunny - <https://github.com/ruby-amqp/bunny>
+class LogStash::Inputs::RabbitMQ < LogStash::Inputs::Threadable
+
+  config_name "rabbitmq"
+  milestone 1
+
+  #
+  # Connection
+  #
+
+  # RabbitMQ server address
+  config :host, :validate => :string, :required => true
+
+  # RabbitMQ port to connect on
+  config :port, :validate => :number, :default => 5672
+
+  # RabbitMQ username
+  config :user, :validate => :string, :default => "guest"
+
+  # RabbitMQ password
+  config :password, :validate => :password, :default => "guest"
+
+  # The vhost to use. If you don't know what this is, leave the default.
+  config :vhost, :validate => :string, :default => "/"
+
+  # Enable or disable SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # Validate SSL certificate
+  config :verify_ssl, :validate => :boolean, :default => false
+
+  # Enable or disable logging
+  config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
+
+
+
+  #
+  # Queue & Consumer
+  #
+
+  # The name of the queue Logstash will consume events from.
+  config :queue, :validate => :string, :default => ""
+
+  # Is this queue durable? (aka; Should it survive a broker restart?)
+  config :durable, :validate => :boolean, :default => false
+
+  # Should the queue be deleted on the broker when the last consumer
+  # disconnects? Set this option to `false` if you want the queue to remain
+  # on the broker, queueing up messages until a consumer comes along to
+  # consume them.
+  config :auto_delete, :validate => :boolean, :default => false
+
+  # Is the queue exclusive? Exclusive queues can only be used by the connection
+  # that declared them and will be deleted when it is closed (e.g. due to a Logstash
+  # restart).
+  config :exclusive, :validate => :boolean, :default => false
+
+  # Extra queue arguments as an array.
+  # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
+  config :arguments, :validate => :array, :default => {}
+
+  # Prefetch count. Number of messages to prefetch
+  config :prefetch_count, :validate => :number, :default => 256
+
+  # Enable message acknowledgement
+  config :ack, :validate => :boolean, :default => true
+
+  # Passive queue creation? Useful for checking queue existance without modifying server state
+  config :passive, :validate => :boolean, :default => false
+
+
+
+  #
+  # (Optional) Exchange binding
+  #
+
+  # Optional.
+  #
+  # The name of the exchange to bind the queue to.
+  config :exchange, :validate => :string
+
+  # Optional.
+  #
+  # The routing key to use when binding a queue to the exchange.
+  # This is only relevant for direct or topic exchanges.
+  #
+  # * Routing keys are ignored on fanout exchanges.
+  # * Wildcards are not valid on direct exchanges.
+  config :key, :validate => :string, :default => "logstash"
+
+
+  def initialize(params)
+    params["codec"] = "json" if !params["codec"]
+
+    super
+  end
+
+  # Use March Hare on JRuby to avoid `IO#select` CPU spikes
+  # (see github.com/ruby-amqp/bunny/issues/95).
+  #
+  # On MRI, use Bunny.
+  #
+  # See http://rubybunny.info and http://rubymarchhare.info
+  # for the docs.
+  if RUBY_ENGINE == "jruby"
+    require "logstash/inputs/rabbitmq/march_hare"
+
+    include MarchHareImpl
+  else
+    require "logstash/inputs/rabbitmq/bunny"
+
+    include BunnyImpl
+  end
+end # class LogStash::Inputs::RabbitMQ

data/lib/logstash/inputs/rabbitmq/bunny.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+class LogStash::Inputs::RabbitMQ
+  module BunnyImpl
+    def register
+      require "bunny"
+
+      @vhost       ||= Bunny::DEFAULT_HOST
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= AMQ::Protocol::DEFAULT_PORT
+      @routing_key ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :automatically_recover => false
+      }
+      @settings[:user]      = @user || Bunny::DEFAULT_USER
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                Bunny::DEFAULT_PASSWORD
+                              end
+
+      @settings[:log_level] = if @debug || @logger.debug?
+                                :debug
+                              else
+                                :error
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      @settings[:verify_ssl] = @verify_ssl if @verify_ssl
+
+      proto                  = if @ssl
+                                 "amqps"
+                               else
+                                 "amqp"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      @logger.info("Registering input #{@connection_url}")
+    end
+
+    def run(output_queue)
+      @output_queue = output_queue
+
+      begin
+        setup
+        consume
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        # Because we manually reconnect instead of letting Bunny
+        # handle failures,
+        # make sure we don't leave any consumer work pool
+        # threads behind. MK.
+        @ch.maybe_kill_consumer_work_pool!
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
+
+        sleep n
+        retry
+      end
+    end
+
+    def teardown
+      @consumer.cancel
+
+      @ch.close   if @ch && @ch.open?
+      @conn.close if @conn && @conn.open?
+
+      finished
+    end
+
+    def setup
+      @conn = Bunny.new(@settings)
+
+      @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+      return if terminating?
+      @conn.start
+
+      @ch = @conn.create_channel.tap do |ch|
+        ch.prefetch(@prefetch_count)
+      end
+      @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+
+      @arguments_hash = Hash[*@arguments]
+
+      @q = @ch.queue(@queue,
+                     :durable     => @durable,
+                     :auto_delete => @auto_delete,
+                     :exclusive   => @exclusive,
+                     :passive     => @passive,
+                     :arguments   => @arguments)
+
+      # exchange binding is optional for the input
+      if @exchange
+        @q.bind(@exchange, :routing_key => @key)
+      end
+    end
+
+    def consume
+      @logger.info("Will consume events from queue #{@q.name}")
+
+      # we both need to block the caller in Bunny::Queue#subscribe and have
+      # a reference to the consumer so that we can cancel it, so
+      # a consumer manually. MK.
+      @consumer = Bunny::Consumer.new(@ch, @q)
+      @q.subscribe(:manual_ack => @ack, :block => true) do |delivery_info, properties, data|
+        @codec.decode(data) do |event|
+          decorate(event)
+          @output_queue << event
+        end
+
+        @ch.acknowledge(delivery_info.delivery_tag) if @ack
+      end
+    end
+  end # BunnyImpl
+end

data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb

@@ -0,0 +1 @@
+require "logstash/inputs/rabbitmq/march_hare"

data/lib/logstash/inputs/rabbitmq/march_hare.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+class LogStash::Inputs::RabbitMQ
+  # MarchHare-based implementation for JRuby
+  module MarchHareImpl
+    def register
+      require "hot_bunnies"
+      require "java"
+
+      @vhost       ||= "127.0.0.1"
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= 5672
+      @key         ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :user  => @user,
+        :automatic_recovery => false
+      }
+      @settings[:pass]      = @password.value if @password
+      @settings[:tls]       = @ssl if @ssl
+
+      proto                 = if @ssl
+                                "amqps"
+                              else
+                                "amqp"
+                              end
+      @connection_url       = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      @logger.info("Registering input #{@connection_url}")
+    end
+
+    def run(output_queue)
+      @output_queue          = output_queue
+      @break_out_of_the_loop = java.util.concurrent.atomic.AtomicBoolean.new(false)
+
+      # MarchHare does not raise exceptions when connection goes down with a blocking
+      # consumer running (it uses callbacks, as the RabbitMQ Java client does).
+      #
+      # However, MarchHare::Channel will make sure to unblock all blocking consumers
+      # on any internal shutdown, so #consume will return and another loop iteration
+      # will run.
+      #
+      # This is very similar to how the Bunny implementation works and is sufficient
+      # for our needs: it recovers successfully after RabbitMQ is kill -9ed, the
+      # network device is shut down, etc. MK.
+      until @break_out_of_the_loop.get do
+        begin
+          setup
+          consume
+        rescue MarchHare::Exception, java.lang.Throwable, com.rabbitmq.client.AlreadyClosedException => e
+          n = 10
+          @logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
+
+          sleep n
+          retry
+        rescue LogStash::ShutdownSignal => ss
+          shutdown_consumer
+        end
+
+        n = 10
+        @logger.error("RabbitMQ connection error: #{e}. Will reconnect in #{n} seconds...")
+      end
+    end
+
+    def teardown
+      shutdown_consumer
+
+      @ch.close         if @ch && @ch.open?
+      @connection.close if @connection && @connection.open?
+
+      finished
+    end
+
+    #
+    # Implementation
+    #
+
+    protected
+
+    def setup
+      return if terminating?
+
+      @conn = MarchHare.connect(@settings)
+      @logger.info("Connected to RabbitMQ #{@connection_url}")
+
+      @ch          = @conn.create_channel.tap do |ch|
+        ch.prefetch = @prefetch_count
+      end
+
+      @arguments_hash = Hash[*@arguments]
+
+      @q = @ch.queue(@queue,
+        :durable     => @durable,
+        :auto_delete => @auto_delete,
+        :exclusive   => @exclusive,
+        :passive     => @passive,
+        :arguments   => @arguments)
+
+      # exchange binding is optional for the input
+      if @exchange
+        @q.bind(@exchange, :routing_key => @key)
+      end
+    end
+
+    def consume
+      return if terminating?
+
+      # we manually build a consumer here to be able to keep a reference to it
+      # in an @ivar even though we use a blocking version of HB::Queue#subscribe
+      @consumer = @q.build_consumer(:block => true) do |metadata, data|
+        @codec.decode(data) do |event|
+          decorate(event)
+          @output_queue << event if event
+        end
+        @ch.ack(metadata.delivery_tag) if @ack
+      end
+      @q.subscribe_with(@consumer, :manual_ack => @ack, :block => true)
+    end
+
+    def shutdown_consumer
+      @break_out_of_the_loop.set(true)
+
+      @consumer.cancel
+      @consumer.gracefully_shut_down
+    end
+  end # MarchHareImpl
+end

data/logstash-input-rabbitmq.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-rabbitmq'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Pull events from a RabbitMQ exchange."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency 'logstash-codec-json'
+
+  if RUBY_PLATFORM == 'java'
+    s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency 'march_hare', ['~> 2.5.1']
+  else
+    s.add_runtime_dependency 'bunny', ['>= 1.6.0']
+  end
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/inputs/rabbitmq_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-rabbitmq
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: java
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: march_hare
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/inputs/rabbitmq.rb
+- lib/logstash/inputs/rabbitmq/bunny.rb
+- lib/logstash/inputs/rabbitmq/hot_bunnies.rb
+- lib/logstash/inputs/rabbitmq/march_hare.rb
+- logstash-input-rabbitmq.gemspec
+- spec/inputs/rabbitmq_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Pull events from a RabbitMQ exchange.
+test_files:
+- spec/inputs/rabbitmq_spec.rb