logstash-input-rabbitmq 3.3.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 234eccd39f2d2459784e232bfee4aca303c51956
-  data.tar.gz: 8ff9cd86fbd139dd7a3623316f1b60749a9bfe57
+  metadata.gz: 9e02bbaff26642c67faaa699f6811a8325838265
+  data.tar.gz: f780dae010e2e3c514f373f079a379156c380a90
 SHA512:
-  metadata.gz: 8e7947a49fc82d0066c9d0563b2dd9ca16d5780ffec7bca130b58a5e1776c3ca7bfaf18528ca1e3f4982ae794f0636e0d029d7e330d6ec9e207bd3117813518a
-  data.tar.gz: efb6b29abfdd37f47cf9ea707789561ab5848f2c00e1e59dc66958802527ef793d871b264cc4d9f86b7757f4c96b1243e14bfd4c7017aa346e2fcd222b3589d3
+  metadata.gz: ff78bf33ae7a6efcbf9228121c2674a2cb6f4c9a674609eb9d1fee36d43aa0a69140e722c10aa860f9ce0cd2f417c71aad3d34bfcc1a0786e55fb8e54cb4ad05
+  data.tar.gz: 504f8ef80becb3557fecf7e7da7945624748356c051f40b06a5d8dd3ad476135efa12b05259e0607e15a1b007d4158aacd7ff3b5413d5d951c6f6b71ad8c6b60

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 4.0.0
+ - Ensure the consumer is done processing messages before closing channel. Fixes shutdown errors.
+ - Use an internal queue + separate thread to accelerate processing
+ - Disable metadata insertion by default, this slows things down majorly
+ - Make exchange_type an optional config option with using 'exchange'. 
+   When used the exchange will always be declared
+
+## 3.3.1
+  - New dependency requirements for logstash-core for the 5.0 release
+
 ## 3.3.0
  - Fix a regression in 3.2.0 that reinstated behavior that duplicated consumers
  - Always declare exchanges used, the exchange now need not exist before LS starts

data/lib/logstash/inputs/rabbitmq.rb

@@ -2,6 +2,8 @@
 require 'logstash/plugin_mixins/rabbitmq_connection'
 require 'logstash/inputs/threadable'
 require 'logstash/event'
+java_import java.util.concurrent.ArrayBlockingQueue
+java_import java.util.concurrent.TimeUnit
 
 module LogStash
   module Inputs
@@ -18,8 +20,9 @@ module LogStash
     # understanding.
     #
     # The properties of messages received will be stored in the
-    # `[@metadata][rabbitmq_properties]` field. The following
-    # properties may be available (in most cases dependent on whether
+    # `[@metadata][rabbitmq_properties]` field if the `@metadata_enabled` setting is checked.
+    # Note that storing metadata may degrade performance.
+    # The following properties may be available (in most cases dependent on whether
     # they were set by the sender):
     #
     # * app-id
@@ -92,6 +95,8 @@ module LogStash
         "user-id",
       ].map { |s| s.freeze }.freeze
 
+      INTERNAL_QUEUE_POISON=[]
+
       config_name "rabbitmq"
 
       # The default codec for this plugin is JSON. You can override this to suit your particular needs however.
@@ -122,8 +127,7 @@ module LogStash
 
       # Prefetch count. If acknowledgements are enabled with the `ack`
       # option, specifies the number of outstanding unacknowledged
-      # messages allowed. With acknowledgemnts disabled this setting
-      # has no effect.
+      # messages allowed.
       config :prefetch_count, :validate => :number, :default => 256
 
       # Enable message acknowledgements. With acknowledgements
@@ -131,6 +135,9 @@ module LogStash
       # Logstash pipeline will be requeued by the server if Logstash
       # shuts down. Acknowledgements will however hurt the message
       # throughput.
+      #
+      # This will only send an ack back every `prefetch_count` messages.
+      # Working in batches provides a performance boost here.
       config :ack, :validate => :boolean, :default => true
 
       # If true the queue will be passively declared, meaning it must
@@ -140,10 +147,12 @@ module LogStash
       # (durable etc) must match those of the existing queue.
       config :passive, :validate => :boolean, :default => false
 
-      # The name of the exchange to bind the queue to.
+      # The name of the exchange to bind the queue to. Specify `exchange_type`
+      # as well to declare the exchange if it does not exist
       config :exchange, :validate => :string
 
-      # The type of the exchange to bind to
+      # The type of the exchange to bind to. Specifying this will cause this plugin
+      # to declare the exchange if it does not exist.
       config :exchange_type, :validate => :string
 
       # The routing key to use when binding a queue to the exchange.
@@ -157,7 +166,12 @@ module LogStash
       # before retrying. Subscribes can fail if the server goes away and then comes back.
       config :subscription_retry_interval_seconds, :validate => :number, :required => true, :default => 5
 
+      # Enable the storage of message headers and properties in `@metadata`. This may impact performance
+      config :metadata_enabled, :validate => :boolean, :default => false
+
       def register
+        @internal_queue = java.util.concurrent.ArrayBlockingQueue.new(@prefetch_count*2)
+
         connect!
         declare_queue!
         bind_exchange!
@@ -178,7 +192,10 @@ module LogStash
 
       def bind_exchange!
         if @exchange
-          @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
+          if @exchange_type # Only declare the exchange if @exchange_type is set!
+            @logger.info? && @logger.info("Declaring exchange '#{@exchange}' with type #{@exchange_type}")
+            @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
+          end
           @hare_info.queue.bind(@exchange, :routing_key => @key)
         end
       end
@@ -197,21 +214,8 @@ module LogStash
       end
 
       def consume!
-        # we manually build a consumer here to be able to keep a reference to it
-        # in an @ivar even though we use a blocking version of HB::Queue#subscribe
-
-        # The logic here around resubscription might seem strange, but its predicated on the fact
-        # that we rely on MarchHare to do the reconnection for us with auto_reconnect.
-        # Unfortunately, while MarchHare does the reconnection work it won't re-subscribe the consumer
-        # hence the logic below.
         @consumer = @hare_info.queue.build_consumer(:on_cancellation => Proc.new { on_cancellation }) do |metadata, data|
-          @codec.decode(data) do |event|
-            decorate(event)
-            event["@metadata"]["rabbitmq_headers"] = get_headers(metadata)
-            event["@metadata"]["rabbitmq_properties"] = get_properties(metadata)
-            @output_queue << event if event
-          end
-          @hare_info.channel.ack(metadata.delivery_tag) if @ack
+          @internal_queue.put [metadata, data]
         end
 
         begin
@@ -223,25 +227,67 @@ module LogStash
           retry
         end
 
-        while !stop?
-          sleep 1
+        internal_queue_consume!
+      end
+
+      def internal_queue_consume!
+        i=0
+        last_delivery_tag=nil
+        while true
+          payload = @internal_queue.poll(10, TimeUnit::MILLISECONDS)
+          if !payload  # Nothing in the queue
+            if last_delivery_tag # And we have unacked stuff
+              @hare_info.channel.ack(last_delivery_tag, true) if @ack
+              i=0
+              last_delivery_tag = nil
+            end
+            next
+          end
+
+          break if payload == INTERNAL_QUEUE_POISON
+
+          metadata, data = payload
+          @codec.decode(data) do |event|
+            decorate(event)
+            if @metadata_enabled
+              event["@metadata"]["rabbitmq_headers"] = get_headers(metadata)
+              event["@metadata"]["rabbitmq_properties"] = get_properties(metadata)
+            end
+            @output_queue << event if event
+          end
+
+          i += 1
+
+          if i >= @prefetch_count
+            @hare_info.channel.ack(metadata.delivery_tag, true) if @ack
+            i = 0
+            last_delivery_tag = nil
+          else
+            last_delivery_tag = metadata.delivery_tag
+          end
         end
       end
 
       def stop
-        super
+        @internal_queue.put(INTERNAL_QUEUE_POISON)
         shutdown_consumer
         close_connection
       end
 
       def shutdown_consumer
         return unless @consumer
-        @consumer.gracefully_shut_down
+        @hare_info.channel.basic_cancel(@consumer.consumer_tag)
+        until @consumer.terminated?
+          @logger.info("Waiting for rabbitmq consumer to terminate before stopping!", :params => self.params)
+          sleep 1
+        end
       end
 
       def on_cancellation
-        @logger.info("Received basic.cancel from #{rabbitmq_settings[:host]}, shutting down.")
-        stop
+        if !stop? # If this isn't already part of a regular stop
+          @logger.info("Received basic.cancel from #{rabbitmq_settings[:host]}, shutting down.")
+          stop
+        end
       end
 
       private

data/logstash-input-rabbitmq.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-rabbitmq'
-  s.version         = '3.3.0'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Pull events from a RabbitMQ exchange."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/rabbitmq_spec.rb

@@ -71,7 +71,7 @@ describe LogStash::Inputs::RabbitMQ do
       context "with an exchange declared" do
         let(:exchange) { "exchange" }
         let(:key) { "routing key" }
-        let(:rabbitmq_settings) { super.merge("exchange" => exchange, "key" => key) }
+        let(:rabbitmq_settings) { super.merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
 
         before do
           allow(instance).to receive(:declare_exchange!)
@@ -180,7 +180,7 @@ describe "with a live server", :integration => true do
   end
 
   describe "receiving a message with a queue + exchange specified" do
-    let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout") }
+    let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => true) }
     let(:event) { output_queue.pop }
     let(:exchange) { test_channel.exchange(exchange_name, :type => "fanout") }
     let(:exchange_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-rabbitmq
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core