logstash-input-rabbitmq 3.1.5 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f93375b1eaa4816cd3abbef1aeaa4f6d4f80045f
-  data.tar.gz: 4f06a024d67fcc9da7e5c8a4a1fb1ee2aa44ea2b
+  metadata.gz: 5caab21a9bb09f1d4af8660f26558c9bde6e9968
+  data.tar.gz: 522d2d66e0c768a06dc14f7dea3e38cc3e4419c0
 SHA512:
-  metadata.gz: a0da602e8f34599aca96f97057a59bb9f2a12b67c5b37c72f5c08a5fdfcf9b5a7f2272840235acd31e4dd994ff1d673a247f339560782db5944a5b41221a8ff7
-  data.tar.gz: 6cc9ab957d7e94ef0dd154a3e3d920d58d4e4dcb603a5589860a60c68f7850eee63ead3aa0a3d08c8cc9178863c64aba44f78d2577481d2141658f654f3c58ed
+  metadata.gz: 1d8a4404784a9de832e23c4b1c9272a87997fbeb4fc5984380c27e6c52a1e8f3661f170e8909eb59bfe7fad15c7af4303e327e44da23ce89632e65a737d1b47b
+  data.tar.gz: 5f01768900d95ddba5259bff24c01856297684c3a46642d78643c96b16d2f508eb37052cb309bc13d8829a6ca3154d97275d85ab96e0c4e22240be6fce2039d7

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.2.0
+ - The properties and headers of the messages are now saved in the [@metadata][rabbitmq_headers] and [@metadata][rabbitmq_properties] fields.
+ - Logstash now shuts down if the server sends a basic.cancel method.
+ - Reinstating the overview documentation that was lost in 3.0.0 and updating it to be clearer.
+ - Internal: Various spec improvements that decrease flakiness and eliminate the queue littering of the integration test RabbitMQ instance.
+
 ## 3.1.5
  - Fix a bug where when reconnecting a duplicate consumer would be created
 

data/lib/logstash/inputs/rabbitmq.rb

@@ -4,15 +4,101 @@ require 'logstash/inputs/threadable'
 
 module LogStash
   module Inputs
+    # Pull events from a http://www.rabbitmq.com/[RabbitMQ] queue.
+    #
+    # The default settings will create an entirely transient queue and listen for all messages by default.
+    # If you need durability or any other advanced settings, please set the appropriate options
+    #
+    # This plugin uses the http://rubymarchhare.info/[March Hare] library
+    # for interacting with the RabbitMQ server. Most configuration options
+    # map directly to standard RabbitMQ and AMQP concepts. The
+    # https://www.rabbitmq.com/amqp-0-9-1-reference.html[AMQP 0-9-1 reference guide]
+    # and other parts of the RabbitMQ documentation are useful for deeper
+    # understanding.
+    #
+    # The properties of messages received will be stored in the
+    # `[@metadata][rabbitmq_properties]` field. The following
+    # properties may be available (in most cases dependent on whether
+    # they were set by the sender):
+    #
+    # * app-id
+    # * cluster-id
+    # * consumer-tag
+    # * content-encoding
+    # * content-type
+    # * correlation-id
+    # * delivery-mode
+    # * exchange
+    # * expiration
+    # * message-id
+    # * priority
+    # * redeliver
+    # * reply-to
+    # * routing-key
+    # * timestamp
+    # * type
+    # * user-id
+    #
+    # For example, to get the RabbitMQ message's timestamp property
+    # into the Logstash event's `@timestamp` field, use the date
+    # filter to parse the `[@metadata][rabbitmq_properties][timestamp]`
+    # field:
+    # [source,ruby]
+    #     filter {
+    #       if [@metadata][rabbitmq_properties][timestamp] {
+    #         date {
+    #           match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]
+    #         }
+    #       }
+    #     }
+    #
+    # Additionally, any message headers will be saved in the
+    # `[@metadata][rabbitmq_headers]` field.
     class RabbitMQ < LogStash::Inputs::Threadable
       include ::LogStash::PluginMixins::RabbitMQConnection
 
+      # The properties to extract from each message and store in a
+      # @metadata field.
+      #
+      # Technically the exchange, redeliver, and routing-key
+      # properties belong to the envelope and not the message but we
+      # ignore that distinction here. However, we extract the
+      # headers separately via get_headers even though the header
+      # table technically is a message property.
+      #
+      # Freezing all strings so that code modifying the event's
+      # @metadata field can't touch them.
+      #
+      # If updating this list, remember to update the documentation
+      # above too.
+      MESSAGE_PROPERTIES = [
+        "app-id",
+        "cluster-id",
+        "consumer-tag",
+        "content-encoding",
+        "content-type",
+        "correlation-id",
+        "delivery-mode",
+        "exchange",
+        "expiration",
+        "message-id",
+        "priority",
+        "redeliver",
+        "reply-to",
+        "routing-key",
+        "timestamp",
+        "type",
+        "user-id",
+      ].map { |s| s.freeze }.freeze
+
       config_name "rabbitmq"
 
       # The default codec for this plugin is JSON. You can override this to suit your particular needs however.
       default :codec, "json"
 
-      # The name of the queue Logstash will consume events from.
+      # The name of the queue Logstash will consume events from. If
+      # left empty, a transient queue with an randomly chosen name
+      # will be created.
       config :queue, :validate => :string, :default => ""
 
       # Is this queue durable? (aka; Should it survive a broker restart?)
@@ -33,13 +119,24 @@ module LogStash
       # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
       config :arguments, :validate => :array, :default => {}
 
-      # Prefetch count. Number of messages to prefetch
+      # Prefetch count. If acknowledgements are enabled with the `ack`
+      # option, specifies the number of outstanding unacknowledged
+      # messages allowed. With acknowledgemnts disabled this setting
+      # has no effect.
       config :prefetch_count, :validate => :number, :default => 256
 
-      # Enable message acknowledgement
+      # Enable message acknowledgements. With acknowledgements
+      # messages fetched by Logstash but not yet sent into the
+      # Logstash pipeline will be requeued by the server if Logstash
+      # shuts down. Acknowledgements will however hurt the message
+      # throughput.
       config :ack, :validate => :boolean, :default => true
 
-      # Passive queue creation? Useful for checking queue existance without modifying server state
+      # If true the queue will be passively declared, meaning it must
+      # already exist on the server. To have Logstash create the queue
+      # if necessary leave this option as false. If actively declaring
+      # a queue that already exists, the queue options for this plugin
+      # (durable etc) must match those of the existing queue.
       config :passive, :validate => :boolean, :default => false
 
       # The name of the exchange to bind the queue to.
@@ -53,7 +150,7 @@ module LogStash
       config :key, :validate => :string, :default => "logstash"
 
       # Amount of time in seconds to wait after a failed subscription request
-      # before retrying. Subscribes can fail if the server goes away and then comes back
+      # before retrying. Subscribes can fail if the server goes away and then comes back.
       config :subscription_retry_interval_seconds, :validate => :number, :required => true, :default => 5
 
       def register
@@ -102,9 +199,12 @@ module LogStash
         # that we rely on MarchHare to do the reconnection for us with auto_reconnect.
         # Unfortunately, while MarchHare does the reconnection work it won't re-subscribe the consumer
         # hence the logic below.
-        @consumer = @hare_info.queue.build_consumer() do |metadata, data|
+        @consumer = @hare_info.queue.build_consumer(:block => true,
+                                                    :on_cancellation => Proc.new { on_cancellation }) do |metadata, data|
           @codec.decode(data) do |event|
             decorate(event)
+            event["@metadata"]["rabbitmq_headers"] = get_headers(metadata)
+            event["@metadata"]["rabbitmq_properties"] = get_properties(metadata)
             @output_queue << event if event
           end
           @hare_info.channel.ack(metadata.delivery_tag) if @ack
@@ -135,6 +235,59 @@ module LogStash
         @consumer.gracefully_shut_down
       end
 
+      def on_cancellation
+        @logger.info("Received basic.cancel from #{rabbitmq_settings[:host]}, shutting down.")
+        stop
+      end
+
+      private
+
+      # ByteArrayLongString is a private static inner class which
+      # can't be access via the regular Java::SomeNameSpace::Classname
+      # notation. See https://github.com/jruby/jruby/issues/3333.
+      ByteArrayLongString = JavaUtilities::get_proxy_class('com.rabbitmq.client.impl.LongStringHelper$ByteArrayLongString')
+
+      def get_header_value(value)
+        # Two kinds of values require exceptional treatment:
+        #
+        # String values are instances of
+        # com.rabbitmq.client.impl.LongStringHelper.ByteArrayLongString
+        # and we don't want to propagate those.
+        #
+        # List values are java.util.ArrayList objects and we need to
+        # recurse into them to convert any nested strings values.
+        if value.class == Java::JavaUtil::ArrayList
+          value.map{|item| get_header_value(item) }
+        elsif value.class == ByteArrayLongString
+          value.toString
+        else
+          value
+        end
+      end
+
+      private
+      def get_headers(metadata)
+        if !metadata.headers.nil?
+          Hash[metadata.headers.map {|k, v| [k, get_header_value(v)]}]
+        else
+          {}
+        end
+      end
+
+      private
+      def get_properties(metadata)
+        MESSAGE_PROPERTIES.reduce({}) do |acc, name|
+          # The method names obviously can't contain hyphens.
+          value = metadata.send(name.gsub("-", "_"))
+          if value
+            # The AMQP 0.9.1 timestamp field only has second resolution
+            # so storing milliseconds serves no purpose and might give
+            # the incorrect impression of a higher resolution.
+            acc[name] = name != "timestamp" ? value : value.getTime / 1000
+          end
+          acc
+        end
+      end
     end
   end
 end

data/logstash-input-rabbitmq.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-rabbitmq'
-  s.version         = '3.1.5'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Pull events from a RabbitMQ exchange."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/rabbitmq_spec.rb

@@ -112,7 +112,7 @@ end
 
 describe "with a live server", :integration => true do
   let(:klass) { LogStash::Inputs::RabbitMQ }
-  let(:config) { {"host" => "127.0.0.1"} }
+  let(:config) { {"host" => "127.0.0.1", "auto_delete" => true, "codec" => "plain" } }
   let(:instance) { klass.new(config) }
   let(:hare_info) { instance.instance_variable_get(:@hare_info) }
   let(:output_queue) { Queue.new }
@@ -134,7 +134,7 @@ describe "with a live server", :integration => true do
     # Extra time to make sure the consumer can attach
     # Without this there's a chance the shutdown code will execute
     # before consumption begins. This is tricky to do more elegantly
-    sleep 1
+    sleep 4
   end
 
   let(:test_connection) { MarchHare.connect(instance.send(:rabbitmq_settings)) }
@@ -166,21 +166,85 @@ describe "with a live server", :integration => true do
   end
 
   describe "receiving a message with a queue specified" do
-    let(:queue_name) { "foo_queue" }
     let(:config) { super.merge("queue" => queue_name) }
+    let(:event) { output_queue.pop }
+    let(:queue) { test_channel.queue(queue_name, :auto_delete => true) }
+    let(:queue_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
 
-    it "should process the message" do
-      message = "Foo Message"
-      q = test_channel.queue(queue_name)
-      q.publish(message)
+    context "when the message has a payload but no message headers" do
+      before do
+        queue.publish(message)
+      end
+
+      let(:message) { "Foo Message" }
+
+      it "should process the message and store the payload" do
+        expect(event["message"]).to eql(message)
+      end
 
-      event = output_queue.pop
-      expect(event["message"]).to eql(message)
+      it "should save an empty message header hash" do
+        expect(event).to include("@metadata")
+        expect(event["@metadata"]).to include("rabbitmq_headers")
+        expect(event["@metadata"]["rabbitmq_headers"]).to eq({})
+      end
+    end
+
+    context "when message properties are available" do
+      before do
+        # Don't test every single property but select a few with
+        # different characteristics to get sufficient coverage.
+        queue.publish("",
+                      :properties => {
+                        :app_id    => app_id,
+                        :timestamp => Java::JavaUtil::Date.new(epoch * 1000),
+                        :priority  => priority,
+                      })
+      end
+
+      let(:app_id) { "myapplication" }
+      # Randomize the epoch we test with but limit its range to signed
+      # ints to not assume all protocols and libraries involved use
+      # unsigned ints for epoch values.
+      let(:epoch) { rand(0x7FFFFFFF) }
+      let(:priority) { 5 }
+
+      it "should save message properties into a @metadata field" do
+        expect(event).to include("@metadata")
+        expect(event["@metadata"]).to include("rabbitmq_properties")
+
+        props = event["@metadata"]["rabbitmq_properties"]
+        expect(props["app-id"]).to eq(app_id)
+        expect(props["delivery-mode"]).to eq(1)
+        expect(props["exchange"]).to eq("")
+        expect(props["priority"]).to eq(priority)
+        expect(props["routing-key"]).to eq(queue_name)
+        expect(props["timestamp"]).to eq(epoch)
+      end
+    end
+
+    context "when message headers are available" do
+      before do
+        queue.publish("", :properties => { :headers => headers })
+      end
+
+      let (:headers) {
+        {
+          "arrayvalue"  => [true, 123, "foo"],
+          "boolvalue"   => true,
+          "intvalue"    => 123,
+          "stringvalue" => "foo",
+        }
+      }
+
+      it "should save message headers into a @metadata field" do
+        expect(event).to include("@metadata")
+        expect(event["@metadata"]).to include("rabbitmq_headers")
+        expect(event["@metadata"]["rabbitmq_headers"]).to include(headers)
+      end
     end
   end
 
   describe LogStash::Inputs::RabbitMQ do
-    let(:config) { super.merge("queue" => "foo_queue") }
     it_behaves_like "an interruptible input plugin" do
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-rabbitmq
 version: !ruby/object:Gem::Version
-  version: 3.1.5
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-28 00:00:00.000000000 Z
+date: 2016-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core