logstash-input-bunny 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a3d026a10ffe2b4a072e4a5584ef2bd1003db28
-  data.tar.gz: fd96ffef182ced185b47afffaf8a6693272c04db
+  metadata.gz: b5c4dbbe89ddbb17aeacca3faecf3647eecced04
+  data.tar.gz: 645754ffc7b545fd97188364d5e556e1fd0474c1
 SHA512:
-  metadata.gz: 478a7b649be84c2b471638313432e8af954d2901aaa748c4c30c4f3324423d39d4fc4d5ff0bbf432723061e9a2eb97e1560bedfbca32c0fc23587e8a1202201c
-  data.tar.gz: 0510b6343762f258e8ad974d1d560d378fe829da547810df178b5c24595073b8f4082ab2606a2c588a62cd79b517e2a47c6463f03d2115ed6fb07a8ea01cdbe8
+  metadata.gz: f64cf8c7e46897dee3c0245452045deb1589f1bd7d08e9e1f21ab8c78848177ccb6876d67a32cf99c342f787f04b66fa7e8bc71f275392285d27fd6a7e454bc4
+  data.tar.gz: b2371829cd2ea89ea4d5e8c9e51f40414bf1cac3dca0ed7bcaf96b658a29715e35b51ced95b581fd2a74b2b9d72854c53cac3804378408309441b5eca8437740

data/lib/logstash/inputs/bunny.rb

@@ -13,222 +13,213 @@ require "logstash/inputs/rabbitmq/bunny"
 #
 # * Bunny - <https://github.com/ruby-amqp/bunny>
 class LogStash::Inputs::Bunny < LogStash::Inputs::Threadable
-
   config_name "bunny"
 
   #
   # Connection
   #
 
-  # RabbitMQ server address
-  config :host, :validate => :string, :required => true
-
-  # RabbitMQ port to connect on
-  config :port, :validate => :number, :default => 5672
-
-  # RabbitMQ username
-  config :user, :validate => :string, :default => "guest"
-
-  # RabbitMQ password
-  config :password, :validate => :password, :default => "guest"
+  config :host,         validate: :string, required: true
+  config :port,         validate: :number, default: 5672
+  config :user,         validate: :string, default: "guest"
+  config :password,     validate: :password, default: "guest"
+  config :vhost,        validate: :string, default: "/"
 
-  # The vhost to use. If you don't know what this is, leave the default.
-  config :vhost, :validate => :string, :default => "/"
+  config :ssl,          validate: :boolean, :default => false
+  config :ssl_cert,     validate: :string
+  config :ssl_key,      validate: :string
+  config :ssl_ca_cert,  validate: :string
+  config :verify_peer,  validate: :boolean, default: false
 
-  # Enable or disable SSL
-  config :ssl,          :validate => :boolean, :default => false
-  config :ssl_cert,     :validate => :string
-  config :ssl_key,      :validate => :string
-  config :ssl_ca_cert,  :validate => :string
-
-  # Validate SSL certificate
-  config :verify_ssl,   :validate => :boolean, :default => false
-  config :verify_peer,  :validate => :boolean, :default => false
+  #
+  # Queue & Consumer
+  #
 
-  # Enable or disable logging
-  config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
+  #
+  # The Bunny plugin's approach differs from Logstash's RabbitMQ plugin in that
+  # we require an exchange so that we can create & bind multiple queues based
+  # on routing keys.
+  #
+  # In addition, rather than accepting a single string for the queue, we now
+  # accept a hash of queue > routing_key pairs. 
+  #
 
+  config :exchange,     validate: :string, required: true
+  config :queues,       validate: :hash, required: true 
 
+  config :prefetch,     validate: :number, default: 20
+  config :ack,          validate: :boolean, default: true
 
   #
-  # Queue & Consumer
+  # Queue Durability / Persistence
   #
 
-  # The name of the queue Logstash will consume events from.
-  config :queue, :validate => :string, :default => ""
+  # Create durable queues so that they survive broker restarts
+  config :durable,      validate: :boolean, default: false
 
-  # Is this queue durable? (aka; Should it survive a broker restart?)
-  config :durable, :validate => :boolean, :default => false
-
-  # Should the queue be deleted on the broker when the last consumer
-  # disconnects? Set this option to `false` if you want the queue to remain
-  # on the broker, queueing up messages until a consumer comes along to
-  # consume them.
-  config :auto_delete, :validate => :boolean, :default => false
+  # Delete queues once the last consumer disconnects.
+  config :auto_delete,  validate: :boolean, default: false
 
   # Is the queue exclusive? Exclusive queues can only be used by the connection
   # that declared them and will be deleted when it is closed (e.g. due to a Logstash
   # restart).
-  config :exclusive, :validate => :boolean, :default => false
-
-  # Extra queue arguments as an array.
-  # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
-  config :arguments, :validate => :array, :default => {}
+  config :exclusive,    validate: :boolean, default: false
 
-  # Prefetch count. Number of messages to prefetch
-  config :prefetch_count, :validate => :number, :default => 256
+  # Passive queue creation? Useful for checking queue existance without modifying server state
+  config :passive,      validate: :boolean, default: false
 
-  # Enable message acknowledgement
-  config :ack, :validate => :boolean, :default => true
+  #
+  # Initialization
+  #
 
-  # Passive queue creation? Useful for checking queue existance without modifying server state
-  config :passive, :validate => :boolean, :default => false
+  def initialize(params)
+    params["codec"] = "json" if !params["codec"]
+    super
+  end
 
+  def input_tag
+    "#{@vhost}/#{@exchange}"
+  end
 
+  def message(severity, message, data={})
+    message = "Bunny[#{input_tag}] #{message}"
+    @logger.send(severity, message, data)
+  end
 
-  #
-  # (Optional) Exchange binding
-  #
+  def register
+    require "bunny"
 
-  # Optional.
-  #
-  # The name of the exchange to bind the queue to.
-  config :exchange, :validate => :string
+    message(:info, "registering input", host: @host)
 
-  # Optional.
-  #
-  # The routing key to use when binding a queue to the exchange.
-  # This is only relevant for direct or topic exchanges.
-  #
-  # * Routing keys are ignored on fanout exchanges.
-  # * Wildcards are not valid on direct exchanges.
-  config :key, :validate => :string, :default => "logstash"
+    @vhost       ||= Bunny::DEFAULT_HOST
+    @port        ||= AMQ::Protocol::DEFAULT_PORT
 
+    @settings = {
+      vhost: @vhost,
+      host:  @host,
+      port:  @port,
+      user:  @user,
+      pass:  @password.value,
+      tls:   @ssl,
+      automatically_recover: false
+    }
 
-  def initialize(params)
-    params["codec"] = "json" if !params["codec"]
+    if @ssl 
+      @settings[:verify_peer] = @verify_peer
+    end
 
-    super
+    if @ssl and @ssl_cert
+      @settings[:tls_cert]            = @ssl_cert
+      @settings[:tls_key]             = @ssl_key
+      @settings[:tls_ca_certificates] = [@ssl_ca_cert] if @ssl_ca_cert
+    end
   end
   
-  include ::LogStash::Inputs::RabbitMQ::BunnyImpl
-
   def run(output_queue)
     @output_queue = output_queue
+    delay = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
 
     begin
       setup
       consume
-    rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, 
-           Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
-      n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
-
-      # Because we manually reconnect instead of letting Bunny
-      # handle failures,
-      # make sure we don't leave any consumer work pool
-      # threads behind. MK.
-      @ch.maybe_kill_consumer_work_pool!
-      @logger.error("RabbitMQ[#{input_name}] connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
-
-      sleep n
-      retry
-
-    rescue OpenSSL::SSL::SSLError => e
-      n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+    rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError,
+           Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed,
+           OpenSSL::SSL::SSLError => e
+
+      @consumers.each do |name, consumer|
+        error = "connection error: #{e.message}. Retrying in #{delay} seconds."
+        message(:error, error)
+        consumer.channel.maybe_kill_consumer_work_pool!
+      end
 
-      @logger.error("RabbitMQ[#{input_name}] SSL connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...")
-      sleep n
+      sleep delay
       retry
 
     rescue LogStash::ShutdownSignal
-      # ignore and quit
-    
-    rescue Exception => e
-      @logger.error("RabbitMQ[#{input_name}] unhandled exception: #{e.inspect}")
-      @logger.error(e.backtrace)
-    end
-  end
+      release
 
-  def register
-    super
+    rescue Exception => e
+      message(:error, "unhandled exception: #{e.inspect}", backtrace: e.backtrace)
 
-    if @ssl and @ssl_cert
-      @settings[:tls_cert]             = @ssl_cert
-      @settings[:tls_key]              = @ssl_key
-      @settings[:tls_ca_certificates]  = [@ssl_ca_cert] if @ssl_ca_cert
-      @settings[:verify_peer]          = @verify_peer
     end
   end
 
   def teardown
-    @consumer.cancel if @consumer
-    @ch.close   if @ch && @ch.open?
-    @conn.close if @conn && @conn.open?
+    release
     finished
   end
 
-  protected
-
-  def consumer_id
-    @consumer_id ||= "#{`hostname -s`.strip}-#{Thread.current.object_id}"
-  end
-
-  def input_name
-    @exchange.nil? or @key.nil? ? queue : "#{@exchange}:#{@key}"
-  end
-
   def setup
     return if terminating?
 
+    #
+    # With multiple threads, Logstash/Jruby/Bunny combo seems to fail ssl
+    # handshake on the first 1-2 connection attempts when restarting Logstash.
+    #
+    # optionally, look at : settings[:connect_timeout]
+    #
     begin
-      attempts ||= 3
-      
-      @logger.debug("RabbitMQ[#{input_name}] connecting.")
-      @conn = Bunny.new(@settings)
-      @conn.start
+      connection_attempt ||= 1
+      message(:debug, "connecting")
+      @connection = Bunny.new(@settings)
+      @connection.start
     rescue OpenSSL::SSL::SSLError
-      @logger.warn("SSL Handshake failed, retrying.")
-      attempts -= 1
-      retry unless attempts == 0
+      message(:warn, "ssl handshake failed, retrying", 
+              attempt: connection_attempt)
+      connection_attempt += 1
+      retry unless connection_attempt > 3
     end
 
-    @logger.info("RabbitMQ[#{input_name}] connected at #{@settings[:host]}")
-    @ch = @conn.create_channel.tap do |ch|
-      ch.prefetch(@prefetch_count)
-    end
+    @consumers = {}
+    @queues.each do |queue_name, routing_key|
+      consumer = "#{queue_name}#{routing_key}"
+      message(:debug, "creating consumer #{consumer}")
+
+      channel = @connection.create_channel.tap do |channel|
+        channel.prefetch(@prefetch)
+      end
+      
+      queue = channel.queue(@queue,
+                            durable: @durable,
+                            auto_delete: @auto_delete,
+                            exclusive: @exclusive,
+                            passive: @passive)
+
+      queue.bind(@exchange, routing_key: routing_key)
 
-    @arguments_hash = Hash[*@arguments]
-    @q = @ch.queue(@queue,
-           :durable     => @durable,
-           :auto_delete => @auto_delete,
-           :exclusive   => @exclusive,
-           :passive     => @passive,
-           :arguments   => @arguments)
-
-    # exchange binding is optional for the input
-    if @exchange
-      @q.bind(@exchange, :routing_key => @key)
+      @consumers[consumer] = Bunny::Consumer.
+                              new(channel, queue, nil, !@ack, @exclusive)
     end
   end
 
-  def consume
-    @logger.info("Will consume events from queue #{@q.name} as #{consumer_id}")
+  def release
+    @consumers.each do |name, consumer|
+      consumer.cancel
+      consumer.channel.close if consumer.channel and consumer.channel.open?
+    end
+    @consumers = []
 
-    no_ack = !@ack
-    exclusive = @exclusive
+    @connection.close if @connection and @connection.open?
+  end
 
-    @consumer = Bunny::Consumer.new(@ch, @q, consumer_id, no_ack, exclusive)
-    @consumer.on_delivery do |delivery_info, properties, data|
-      @codec.decode(data) do |event|
-        decorate(event)
-        @output_queue << event
+  def consume
+    @consumers.each do |name, consumer|
+      message(:info, "consuming events from #{name}")
+
+      consumer.on_delivery do |delivery_info, properties, data|
+        @codec.decode(data) do |event|
+          decorate(event)
+          event["bunny"] ||= {}
+          event["bunny"]["key"] = delivery_info.routing_key
+          event["bunny"]["queue"] = consumer.queue_name
+          @output_queue << event
+        end
+
+        consumer.channel.acknowledge(delivery_info.delivery_tag) if @ack
       end
 
-      @ch.acknowledge(delivery_info.delivery_tag) if @ack
+      consumer.queue.subscribe_with(consumer, block: true)
     end
-
-    @q.subscribe_with(@consumer, block: true)
   end
-  
 end # class LogStash::Inputs::RabbitMQ
 

data/logstash-input-bunny.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-bunny'
-  s.version         = '0.1.11'
+  s.version         = '0.1.12'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Pull events from a RabbitMQ exchange."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-bunny
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Jonathan Serafini
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-07 00:00:00.000000000 Z
+date: 2015-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core