logstash-integration-rabbitmq 7.0.0-java → 7.1.1-java

data/docs/output-rabbitmq.asciidoc

@@ -1,3 +1,4 @@
+:integration: rabbitmq
 :plugin: rabbitmq
 :type: output
 :default_codec: json
@@ -17,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Rabbitmq output plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
@@ -168,7 +169,8 @@ Key to route to by default. Defaults to 'logstash'
   * Value type is <<hash,hash>>
   * Default value is `{}`
 
-Add properties to be set per-message here, such as 'content_type', 'priority'
+Add properties to be set per-message here, such as 'content_type', 'priority'.
+Values can be {logstash-ref}/event-dependent-configuration.html#sprintf[`sprintf` templates], whose value for each message will be populated from the event.
 
 Example:
 [source,ruby]

data/lib/logstash/inputs/rabbitmq.rb

@@ -2,8 +2,6 @@
 require_relative '../plugin_mixins/rabbitmq_connection'
 require 'logstash/inputs/threadable'
 require 'logstash/event'
-java_import java.util.concurrent.ArrayBlockingQueue
-java_import java.util.concurrent.TimeUnit
 
 module LogStash
   module Inputs
@@ -59,6 +57,9 @@ module LogStash
     # Additionally, any message headers will be saved in the
     # `[@metadata][rabbitmq_headers]` field.
     class RabbitMQ < LogStash::Inputs::Threadable
+
+      java_import java.util.concurrent.TimeUnit
+
       include ::LogStash::PluginMixins::RabbitMQConnection
 
       # The properties to extract from each message and store in a
@@ -121,8 +122,6 @@ module LogStash
       # restart).
       config :exclusive, :validate => :boolean, :default => false
 
-      # Extra queue arguments as an array.
-      # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
       config :arguments, :validate => :array, :default => {}
 
       # Prefetch count. If acknowledgements are enabled with the `ack`
@@ -167,16 +166,26 @@ module LogStash
       config :subscription_retry_interval_seconds, :validate => :number, :required => true, :default => 5
 
       # Enable the storage of message headers and properties in `@metadata`. This may impact performance
-      config :metadata_enabled, :validate => :boolean, :default => false
+      config :metadata_enabled, :validate => %w(none basic extended false true), :default => "none"
 
       def register
         @internal_queue = java.util.concurrent.ArrayBlockingQueue.new(@prefetch_count*2)
+        @metadata_level = extract_metadata_level(@metadata_enabled)
       end
 
+      attr_reader :metadata_level
+
       def run(output_queue)
         setup!
         @output_queue = output_queue
         consume!
+      rescue => e
+        raise unless stop?
+
+        logger.warn("Ignoring exception thrown during plugin shutdown",
+                    :message  => e.message,
+                    :class    => e.class.name,
+                    :location => e.backtrace.first)
       end
 
       def setup!
@@ -185,14 +194,32 @@ module LogStash
         bind_exchange!
         @hare_info.channel.prefetch = @prefetch_count
       rescue => e
+        # when encountering an exception during shut-down,
+        # re-raise the exception instead of retrying
+        raise if stop?
+
+        reset!
+
         @logger.warn("Error while setting up connection for rabbitmq input! Will retry.",
-                     :message => e.message,
-                     :class => e.class.name,
+                     :message  => e.message,
+                     :class    => e.class.name,
                      :location => e.backtrace.first)
         sleep_for_retry
         retry
       end
 
+      # reset a partially-established connection, enabling subsequent
+      # call to `RabbitMQ#setup!` to succeed.
+      #
+      # @api private
+      def reset!
+        @hare_info.connection && @hare_info.connection.close
+      rescue => e
+        @logger.debug("Exception while resetting connection", :exception => e.message, :backtrace => e.backtrace)
+      ensure
+        @hare_info = nil
+      end
+
       def bind_exchange!
         if @exchange
           if @exchange_type # Only declare the exchange if @exchange_type is set!
@@ -251,12 +278,11 @@ module LogStash
 
           metadata, data = payload
           @codec.decode(data) do |event|
-            decorate(event)
-            if @metadata_enabled
-              event.set("[@metadata][rabbitmq_headers]", get_headers(metadata))
-              event.set("[@metadata][rabbitmq_properties]", get_properties(metadata))
+            if event
+              decorate(event, metadata, data)
+
+              @output_queue << event
             end
-            @output_queue << event if event
           end
 
           i += 1
@@ -271,6 +297,16 @@ module LogStash
         end
       end
 
+      def decorate(event, metadata, data)
+        super(event)
+
+        event.set("[@metadata][rabbitmq_headers]", get_headers(metadata))       if metadata_level.include?(:headers)
+        event.set("[@metadata][rabbitmq_properties]", get_properties(metadata)) if metadata_level.include?(:properties)
+        event.set("[@metadata][rabbitmq_payload]", data)                        if metadata_level.include?(:payload) && !data.nil?
+
+        nil
+      end
+
       def stop
         @internal_queue.put(INTERNAL_QUEUE_POISON)
         shutdown_consumer
@@ -312,6 +348,28 @@ module LogStash
           acc
         end
       end
+
+      METADATA_NONE     = Set[].freeze
+      METADATA_BASIC    = Set[:headers,:properties].freeze
+      METADATA_EXTENDED = Set[:headers,:properties,:payload].freeze
+      METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+      private
+      def extract_metadata_level(metadata_enabled_setting)
+        metadata_enabled = metadata_enabled_setting
+
+        if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+          canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+          logger.warn("Deprecated value `#{metadata_enabled_setting}` for `metadata_enabled` option; use `#{canonical_value}` instead.")
+          metadata_enabled = canonical_value
+        end
+
+        case metadata_enabled
+        when 'none'     then METADATA_NONE
+        when 'basic'    then METADATA_BASIC
+        when 'extended' then METADATA_EXTENDED
+        end
+      end
     end
   end
 end

data/lib/logstash/outputs/rabbitmq.rb

@@ -1,14 +1,12 @@
 # encoding: UTF-8
 require "logstash/pipeline"
 require_relative '../plugin_mixins/rabbitmq_connection'
-java_import java.util.concurrent.TimeoutException
-java_import com.rabbitmq.client.AlreadyClosedException
 
 require 'back_pressure'
 
 # Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
 # or later version (3.x is recommended).
-# 
+#
 # Relevant links:
 #
 # * http://www.rabbitmq.com/[RabbitMQ]
@@ -16,10 +14,14 @@ require 'back_pressure'
 module LogStash
   module Outputs
     class RabbitMQ < LogStash::Outputs::Base
+
+      java_import java.util.concurrent.TimeoutException
+      java_import com.rabbitmq.client.AlreadyClosedException
+
       include LogStash::PluginMixins::RabbitMQConnection
 
       config_name "rabbitmq"
-      
+
       concurrency :shared
 
       # The default codec for this plugin is JSON. You can override this to suit your particular needs however.
@@ -46,6 +48,8 @@ module LogStash
       config :message_properties, :validate => :hash, :default => {}
 
       def register
+        @message_properties_template = MessagePropertiesTemplate.new(symbolize(@message_properties).merge(:persistent => @persistent))
+
         connect!
         @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
         # The connection close should close all channels, so it is safe to store thread locals here without closing them
@@ -67,8 +71,10 @@ module LogStash
 
       def publish(event, message)
         raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
+        routing_key = event.sprintf(@key)
+        message_properties = @message_properties_template.build(event)
         @gated_executor.execute do
-          local_exchange.publish(message, :routing_key => event.sprintf(@key), :properties => symbolize(@message_properties.merge(:persistent => @persistent)))
+          local_exchange.publish(message, :routing_key => routing_key, :properties => message_properties)
         end
       rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
         @logger.error("Error while publishing. Will retry.",
@@ -118,7 +124,7 @@ module LogStash
           march_hare_connection.on_unblocked do
             executor.remove_back_pressure('connection flagged as unblocked')
           end
-          march_hare_connection.on_recovery_started do
+          march_hare_connection.on_recovery_start do
             executor.engage_back_pressure("connection is being recovered")
           end
           march_hare_connection.on_recovery do
@@ -126,6 +132,64 @@ module LogStash
           end
         end
       end
+
+      ##
+      # A `MessagePropertiesTemplate` efficiently produces per-event message properties from the
+      # provided template Hash.
+      #
+      # In order to efficiently reuse constant-value objects, returned values may be frozen.
+      class MessagePropertiesTemplate
+        ##
+        # Creates a new `MessagePropertiesTemplate` from the provided `template`
+        # @param template [Hash{Symbol=>Object}]
+        def initialize(template)
+          constant_properties = template.reject { |_,v| templated?(v) }
+          variable_properties = template.select { |_,v| templated?(v) }
+
+          @constant_properties = normalize(constant_properties).freeze
+          @variable_properties = variable_properties
+        end
+
+        ##
+        # Builds a property mapping for the given `event`, including templated values.
+        #
+        # @param event [LogStash::Event]: the event with which to populated templated values, if any.
+        # @return [Hash{Symbol=>Object}] a possibly-frozen properties hash for the provided `event`.
+        def build(event)
+          return @constant_properties if @variable_properties.empty?
+
+          properties = @variable_properties.each_with_object(@constant_properties.dup) do |(k,v), memo|
+            memo.store(k, event.sprintf(v))
+          end
+
+          return normalize(properties)
+        end
+
+        private
+
+        ##
+        # Normalize the provided property mapping with respect to the value types the underlying
+        # client expects.
+        #
+        # @api private
+        # @param properties [Hash{Symbol=>Object}]: a possibly-frozen Hash whose values may need type-coercion.
+        # @return [Hash{Symbol=>Object}]
+        def normalize(properties)
+          if properties[:priority] && properties[:priority].kind_of?(String)
+            properties = properties.merge(:priority => properties[:priority].to_i)
+          end
+
+          properties
+        end
+
+        ##
+        # @api private
+        # @param [Object]: an object, which may or may not be a template `String`
+        # @return [Boolean]: returns `true` IFF `value` is a template `String`
+        def templated?(value)
+          value.kind_of?(String) && value.include?('%{')
+        end
+      end
     end
   end
-end
+end

data/lib/logstash/plugin_mixins/rabbitmq_connection.rb

@@ -104,10 +104,10 @@ module LogStash
       def rabbitmq_settings
         return @rabbitmq_settings if @rabbitmq_settings
 
+
         s = {
           :vhost => @vhost,
-          :hosts => @host,
-          :port  => @port,
+          :addresses => addresses_from_hosts_and_port(@host, @port),
           :user  => @user,
           :automatic_recovery => @automatic_recovery,
           :pass => @password ? @password.value : "guest",
@@ -133,6 +133,11 @@ module LogStash
         @rabbitmq_settings = s
       end
 
+      def addresses_from_hosts_and_port(hosts, port)
+        hosts.map {|host| host.include?(':') ? host : "#{host}:#{port}"}
+      end
+
+
       def connect!
         @hare_info = connect() unless @hare_info # Don't duplicate the conn!
       rescue MarchHare::Exception, java.io.IOException => e

data/logstash-integration-rabbitmq.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-rabbitmq'
-  s.version         = '7.0.0'
+  s.version         = '7.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Integration with RabbitMQ - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -47,7 +47,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_runtime_dependency 'back_pressure', '~> 1.0'
 
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', '~>2.0'
   s.add_development_dependency 'logstash-input-generator'
   s.add_development_dependency 'logstash-codec-plain'
 end

data/spec/inputs/rabbitmq_spec.rb

@@ -21,8 +21,13 @@ describe LogStash::Inputs::RabbitMQ do
       "prefetch_count" => 123
     }
   }
-  let(:instance) { klass.new(rabbitmq_settings) }
+  subject(:instance) { klass.new(rabbitmq_settings) }
   let(:hare_info) { instance.instance_variable_get(:@hare_info) }
+  let(:instance_logger) { double("Logger").as_null_object }
+
+  before do
+    allow_any_instance_of(klass).to receive(:logger).and_return(instance_logger)
+  end
 
   context "when connected" do
     let(:connection) { double("MarchHare Connection") }
@@ -39,6 +44,7 @@ describe LogStash::Inputs::RabbitMQ do
       allow(connection).to receive(:on_shutdown)
       allow(connection).to receive(:on_blocked)
       allow(connection).to receive(:on_unblocked)
+      allow(connection).to receive(:close)
       allow(channel).to receive(:exchange).and_return(exchange)
       allow(channel).to receive(:queue).and_return(queue)
       allow(channel).to receive(:prefetch=)
@@ -127,11 +133,193 @@ describe LogStash::Inputs::RabbitMQ do
       end
     end
   end
+
+  context '#register' do
+    let(:rabbitmq_settings) { super().merge(metadata_enabled_override) }
+    let(:metadata_enabled_override) { { "metadata_enabled" => metadata_enabled } }
+    before do
+      instance.register
+    end
+
+    shared_examples('`metadata_enabled => none`') do
+      context 'metadata_level' do
+        subject(:metadata_level) { instance.metadata_level }
+        it { is_expected.to be_empty }
+        it { is_expected.to be_frozen }
+      end
+    end
+
+    shared_examples('`metadata_enabled => basic`') do
+      context 'metadata_level' do
+        subject(:metadata_level) { instance.metadata_level }
+        it { is_expected.to include :headers }
+        it { is_expected.to include :properties }
+        it { is_expected.to_not include :payload }
+        it { is_expected.to be_frozen }
+      end
+    end
+
+    shared_examples("deprecated `metadata_enabled` setting") do |deprecated_value|
+      context 'the logger' do
+        subject(:logger) { instance_logger }
+        it 'receives a useful deprecation warning' do
+          expect(logger).to have_received(:warn).with(/Deprecated value `#{Regexp.escape(deprecated_value)}`/)
+        end
+      end
+    end
+
+    context 'when `metadata_enabled` is `true`' do
+      let(:metadata_enabled) { "true" }
+      it_behaves_like '`metadata_enabled => basic`'
+      include_examples "deprecated `metadata_enabled` setting", "true"
+    end
+
+    context 'when `metadata_enabled` is `false`' do
+      let(:metadata_enabled) { "false" }
+      it_behaves_like '`metadata_enabled => none`'
+      include_examples "deprecated `metadata_enabled` setting", "false"
+    end
+
+    context 'when `metadata_enabled` is not provided' do
+      let(:metadata_enabled_override) { Hash.new }
+      it_behaves_like '`metadata_enabled => none`'
+    end
+
+    context 'when `metadata_enabled` is `basic`' do
+      let(:metadata_enabled) { "basic" }
+      include_examples '`metadata_enabled => basic`'
+    end
+
+    context 'when `metadata_enabled` is `none`' do
+      let(:metadata_enabled) { "none" }
+      include_examples '`metadata_enabled => none`'
+    end
+
+    context 'when `metadata_enabled` is `extended`' do
+      let(:metadata_enabled) { "extended" }
+      context 'metadata_level' do
+        subject(:metadata_level) { instance.metadata_level }
+        it { is_expected.to include :headers }
+        it { is_expected.to include :properties }
+        it { is_expected.to include :payload }
+        it { is_expected.to be_frozen }
+      end
+    end
+  end
+
+  describe "#decorate(event, metadata, data)" do
+    let(:rabbitmq_settings) do
+      super().merge("metadata_enabled" => metadata_enabled)
+    end
+    before(:each) { instance.register }
+
+    let(:metadata) { double("METADATA") }
+    let(:headers) { Hash("header_key"=>"header_value") }
+    let(:properties) { Hash("property_key"=>"property_value") }
+
+    let(:data) { %Q({"message"=>"fubar"}\n) }
+
+    before do
+      allow(instance).to receive(:get_headers).with(metadata).and_return(headers)
+      allow(instance).to receive(:get_properties).with(metadata).and_return(properties)
+    end
+
+    describe 'the decorated event' do
+      subject(:decorated_event) do
+        LogStash::Event.new("message"=>"fubar").tap do |e|
+          instance.decorate(e, metadata, data)
+        end
+      end
+
+      matcher :include_field do |fieldref|
+        match do |event|
+          # setting `@actual` makes failure messages clearer
+          @actual = event.to_hash_with_metadata
+
+          break false unless event.include?(fieldref)
+          break true unless @specific_value
+
+          values_match?(@expected_value, event.get(fieldref))
+        end
+        chain :with_value do |expected_value|
+          @specific_value = true
+          @expected_value = expected_value
+        end
+        description do
+          desc = "include field `#{fieldref}`"
+          desc += " with value matching `#{@expected_value.inspect}`" if @specific_value
+          desc
+        end
+      end
+
+      shared_examples('core decoration features') do
+        let(:rabbitmq_settings) do
+          super().merge("type" => "decorated_type",
+                        "add_field" => {"added_field" => "field_value"})
+        end
+        it 'has been decorated with core decoration features' do
+          expect(decorated_event).to include_field("added_field").with_value("field_value")
+          expect(decorated_event).to include_field("type").with_value("decorated_type")
+        end
+      end
+
+      let(:headers_fieldref) { "[@metadata][rabbitmq_headers]" }
+      let(:properties_fieldref) { "[@metadata][rabbitmq_properties]" }
+      let(:payload_fieldref) { "[@metadata][rabbitmq_payload]" }
+
+      shared_examples('`metadata_enabled => none`') do
+        it { is_expected.to_not include_field(headers_fieldref) }
+        it { is_expected.to_not include_field(properties_fieldref) }
+        it { is_expected.to_not include_field(payload_fieldref) }
+
+        include_examples 'core decoration features'
+      end
+
+      shared_examples('`metadata_enabled => basic`') do
+        it { is_expected.to include_field(headers_fieldref).with_value(headers) }
+        it { is_expected.to include_field(properties_fieldref).with_value(properties) }
+        it { is_expected.to_not include_field(payload_fieldref) }
+
+        include_examples 'core decoration features'
+      end
+
+      context "with `metadata_enabled => none`" do
+        let(:metadata_enabled) { "none" }
+        include_examples '`metadata_enabled => none`'
+      end
+
+      context "with `metadata_enabled => basic`" do
+        let(:metadata_enabled) { "basic" }
+        include_examples '`metadata_enabled => basic`'
+      end
+
+      context 'with `metadata_enabled => extended`' do
+        let(:metadata_enabled) { "extended" }
+        it { is_expected.to include_field(headers_fieldref).with_value(headers) }
+        it { is_expected.to include_field(properties_fieldref).with_value(properties) }
+        it { is_expected.to include_field(payload_fieldref).with_value(data) }
+        include_examples 'core decoration features'
+      end
+
+      # Deprecated alias: false -> none
+      context "with `metadata_enabled => false`" do
+        let(:metadata_enabled) { "false" }
+        it_behaves_like '`metadata_enabled => none`'
+      end
+
+      # Deprecated alias: true -> basic
+      context "with `metadata_enabled => true`" do
+        let(:metadata_enabled) { "true" }
+        it_behaves_like '`metadata_enabled => basic`'
+      end
+    end
+  end
 end
 
-describe "with a live server", :integration => true do
+describe "LogStash::Inputs::RabbitMQ with a live server", :integration => true do
   let(:klass) { LogStash::Inputs::RabbitMQ }
-  let(:config) { {"host" => "127.0.0.1", "auto_delete" => true, "codec" => "plain", "add_field" => {"[@metadata][foo]" => "bar"} } }
+  let(:rabbitmq_host) { ENV["RABBITMQ_HOST"] || "127.0.0.1" }
+  let(:config) { {"host" => rabbitmq_host, "auto_delete" => true, "codec" => "plain", "add_field" => {"[@metadata][foo]" => "bar"} } }
   let(:instance) { klass.new(config) }
   let(:hare_info) { instance.instance_variable_get(:@hare_info) }
   let(:output_queue) { Queue.new }
@@ -185,7 +373,7 @@ describe "with a live server", :integration => true do
   end
 
   describe "receiving a message with a queue + exchange specified" do
-    let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => true) }
+    let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => "true") }
     let(:event) { output_queue.pop }
     let(:exchange) { test_channel.exchange(exchange_name, :type => "fanout") }
     let(:exchange_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
@@ -237,7 +425,7 @@ describe "with a live server", :integration => true do
         expect(event).to include("@metadata")
         expect(event.get("@metadata")).to include("rabbitmq_properties")
 
-        props = event.get("[@metadata][rabbitmq_properties")
+        props = event.get("[@metadata][rabbitmq_properties]")
         expect(props["app-id"]).to eq(app_id)
         expect(props["delivery-mode"]).to eq(1)
         expect(props["exchange"]).to eq(exchange_name)
@@ -274,6 +462,7 @@ describe "with a live server", :integration => true do
   end
 
   describe LogStash::Inputs::RabbitMQ do
+    require "logstash/devutils/rspec/shared_examples"
     it_behaves_like "an interruptible input plugin"
   end
 end