logstash-integration-rabbitmq 7.0.0-java

data/lib/logstash/outputs/rabbitmq.rb

@@ -0,0 +1,131 @@
+# encoding: UTF-8
+require "logstash/pipeline"
+require_relative '../plugin_mixins/rabbitmq_connection'
+java_import java.util.concurrent.TimeoutException
+java_import com.rabbitmq.client.AlreadyClosedException
+
+require 'back_pressure'
+
+# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
+# or later version (3.x is recommended).
+# 
+# Relevant links:
+#
+# * http://www.rabbitmq.com/[RabbitMQ]
+# * http://rubymarchhare.info[March Hare]
+module LogStash
+  module Outputs
+    class RabbitMQ < LogStash::Outputs::Base
+      include LogStash::PluginMixins::RabbitMQConnection
+
+      config_name "rabbitmq"
+      
+      concurrency :shared
+
+      # The default codec for this plugin is JSON. You can override this to suit your particular needs however.
+      default :codec, "json"
+
+      # Key to route to by default. Defaults to 'logstash'
+      #
+      # * Routing keys are ignored on fanout exchanges.
+      config :key, :validate => :string, :default => "logstash"
+
+      # The name of the exchange
+      config :exchange, :validate => :string, :required => true
+
+      # The exchange type (fanout, topic, direct)
+      config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
+
+      # Is this exchange durable? (aka; Should it survive a broker restart?)
+      config :durable, :validate => :boolean, :default => true
+
+      # Should RabbitMQ persist messages to disk?
+      config :persistent, :validate => :boolean, :default => true
+
+      # Properties to be passed along with the message
+      config :message_properties, :validate => :hash, :default => {}
+
+      def register
+        connect!
+        @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
+        # The connection close should close all channels, so it is safe to store thread locals here without closing them
+        @thread_local_channel = java.lang.ThreadLocal.new
+        @thread_local_exchange = java.lang.ThreadLocal.new
+
+        @gated_executor = back_pressure_provider_for_connection(@hare_info.connection)
+      end
+
+      def symbolize(myhash)
+        Hash[myhash.map{|(k,v)| [k.to_sym,v]}]
+      end
+
+      def multi_receive_encoded(events_and_data)
+        events_and_data.each do |event, data|
+          publish(event, data)
+        end
+      end
+
+      def publish(event, message)
+        raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
+        @gated_executor.execute do
+          local_exchange.publish(message, :routing_key => event.sprintf(@key), :properties => symbolize(@message_properties.merge(:persistent => @persistent)))
+        end
+      rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
+        @logger.error("Error while publishing. Will retry.",
+                      :message => e.message,
+                      :exception => e.class,
+                      :backtrace => e.backtrace)
+
+        sleep_for_retry
+        retry
+      end
+
+      def local_exchange
+        exchange = @thread_local_exchange.get
+        if !exchange
+          exchange = declare_exchange!(local_channel, @exchange, @exchange_type, @durable)
+          @thread_local_exchange.set(exchange)
+        end
+        exchange
+      end
+
+      def local_channel
+        channel = @thread_local_channel.get
+        if !channel
+          channel = @hare_info.connection.create_channel
+          @thread_local_channel.set(channel)
+        end
+        channel
+      end
+
+      def close
+        close_connection
+      end
+
+      private
+
+      # When the other end of a RabbitMQ connection is either unwilling or unable to continue reading bytes from
+      # its underlying TCP stream, the connection is flagged as "blocked", but attempts to publish onto exchanges
+      # using the connection will not block in the client.
+      #
+      # Here we hook into notifications of connection-blocked state to set up a `BackPressure::GatedExecutor`,
+      # which is used elsewhere to prevent runaway writes when publishing to an exchange on a blocked connection.
+      def back_pressure_provider_for_connection(march_hare_connection)
+        BackPressure::GatedExecutor.new(description: "RabbitMQ[#{self.id}]", logger: logger).tap do |executor|
+          march_hare_connection.on_blocked do |reason|
+            executor.engage_back_pressure("connection flagged as blocked: `#{reason}`")
+          end
+          march_hare_connection.on_unblocked do
+            executor.remove_back_pressure('connection flagged as unblocked')
+          end
+          march_hare_connection.on_recovery_started do
+            executor.engage_back_pressure("connection is being recovered")
+          end
+          march_hare_connection.on_recovery do
+            executor.remove_back_pressure('connection recovered')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/plugin_mixins/rabbitmq_connection.rb

@@ -0,0 +1,234 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "march_hare"
+require "java"
+
+# Common functionality for the rabbitmq input/output
+module LogStash
+  module PluginMixins
+    module RabbitMQConnection
+      EXCHANGE_TYPES = ["fanout", "direct", "topic", "x-consistent-hash", "x-modulus-hash"]
+
+      HareInfo = Struct.new(:connection, :channel, :exchange, :queue)
+
+      def self.included(base)
+        base.extend(self)
+        base.setup_rabbitmq_connection_config
+      end
+
+      def setup_rabbitmq_connection_config
+        # RabbitMQ server address(es)
+        # host can either be a single host, or a list of hosts
+        # i.e.
+        #   host => "localhost"
+        # or
+        #   host => ["host01", "host02]
+        #
+        # if multiple hosts are provided on the initial connection and any subsequent
+        # recovery attempts of the hosts is chosen at random and connected to.
+        # Note that only one host connection is active at a time.
+        config :host, :validate => :string, :required => true , :list => true
+
+        # RabbitMQ port to connect on
+        config :port, :validate => :number, :default => 5672
+
+        # RabbitMQ username
+        config :user, :validate => :string, :default => "guest"
+
+        # RabbitMQ password
+        config :password, :validate => :password, :default => "guest"
+
+        # The vhost (virtual host) to use. If you don't know what this
+        # is, leave the default. With the exception of the default
+        # vhost ("/"), names of vhosts should not begin with a forward
+        # slash.
+        config :vhost, :validate => :string, :default => "/"
+
+        # Enable or disable SSL.
+        # Note that by default remote certificate verification is off.
+        # Specify ssl_certificate_path and ssl_certificate_password if you need
+        # certificate verification
+        config :ssl, :validate => :boolean
+
+        # Version of the SSL protocol to use.
+        config :ssl_version, :validate => :string, :default => "TLSv1.2"
+
+        config :verify_ssl, :validate => :boolean, :default => false,
+          :obsolete => "This function did not actually function correctly and was removed." +
+                       "If you wish to validate SSL certs use the ssl_certificate_path and ssl_certificate_password options."
+
+        # Path to an SSL certificate in PKCS12 (.p12) format used for verifying the remote host
+        config :ssl_certificate_path, :validate => :path
+
+        # Password for the encrypted PKCS12 (.p12) certificate file specified in ssl_certificate_path
+        config :ssl_certificate_password, :validate => :password
+
+        config :debug, :validate => :boolean, :obsolete => "Use the logstash --debug flag for this instead."
+
+        # Set this to automatically recover from a broken connection. You almost certainly don't want to override this!!!
+        config :automatic_recovery, :validate => :boolean, :default => true
+
+        # Time in seconds to wait before retrying a connection
+        config :connect_retry_interval, :validate => :number, :default => 1
+
+        # The default connection timeout in milliseconds. If not specified the timeout is infinite.
+        config :connection_timeout, :validate => :number
+
+        # Heartbeat delay in seconds. If unspecified no heartbeats will be sent
+        config :heartbeat, :validate => :number
+
+        # Passive queue creation? Useful for checking queue existance without modifying server state
+        config :passive, :validate => :boolean, :default => false
+
+        # TLS certifcate path
+        config :tls_certificate_path, :validate => :path, :obsolete => "This setting is obsolete. Use ssl_certificate_path instead"
+
+        # TLS certificate password
+        config :tls_certificate_password, :validate => :string, :obsolete => "This setting is obsolete. Use ssl_certificate_password instead"
+
+        # Extra queue arguments as an array.
+        # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
+        config :arguments, :validate => :array, :default => {}
+      end
+
+      def conn_str
+        "amqp://#{@user}@#{@host}:#{@port}#{@vhost}"
+      end
+
+      def close_connection
+        @rabbitmq_connection_stopping = true
+        @hare_info.channel.close if channel_open?
+        @hare_info.connection.close if connection_open?
+      end
+
+      def rabbitmq_settings
+        return @rabbitmq_settings if @rabbitmq_settings
+
+        s = {
+          :vhost => @vhost,
+          :hosts => @host,
+          :port  => @port,
+          :user  => @user,
+          :automatic_recovery => @automatic_recovery,
+          :pass => @password ? @password.value : "guest",
+        }
+
+        s[:timeout] = @connection_timeout || 0
+        s[:heartbeat] = @heartbeat || 0
+
+        if @ssl
+          s[:tls] = @ssl_version
+
+          cert_path = @ssl_certificate_path
+          cert_pass = @ssl_certificate_password.value if @ssl_certificate_password
+
+          if !!cert_path ^ !!cert_pass
+            raise LogStash::ConfigurationError, "RabbitMQ requires both ssl_certificate_path AND ssl_certificate_password to be set!"
+          end
+
+          s[:tls_certificate_path] = cert_path
+          s[:tls_certificate_password] = cert_pass
+        end
+
+        @rabbitmq_settings = s
+      end
+
+      def connect!
+        @hare_info = connect() unless @hare_info # Don't duplicate the conn!
+      rescue MarchHare::Exception, java.io.IOException => e
+        error_message = if e.message.empty? && e.is_a?(java.io.IOException)
+          # IOException with an empty message is probably an instance of
+          # these problems:
+          # https://github.com/logstash-plugins/logstash-output-rabbitmq/issues/52
+          # https://github.com/rabbitmq/rabbitmq-java-client/issues/100
+          #
+          # Best guess is to help the user understand that there is probably
+          # some kind of configuration problem causing the error, but we
+          # can't really offer any more detailed hints :\
+          "An unknown error occurred. RabbitMQ gave no hints as to the cause. Maybe this is a configuration error (invalid vhost, for example). I recommend checking the RabbitMQ server logs for clues about this failure."
+        else
+          e.message
+        end
+
+        if @logger.debug?
+          @logger.error("RabbitMQ connection error, will retry.",
+                        :error_message => error_message,
+                        :exception => e.class.name,
+                        :backtrace => e.backtrace)
+        else
+          @logger.error("RabbitMQ connection error, will retry.",
+                        :error_message => error_message,
+                        :exception => e.class.name)
+        end
+
+        sleep_for_retry
+        retry
+      end
+
+      def channel_open?
+        @hare_info && @hare_info.channel && @hare_info.channel.open?
+      end
+
+      def connection_open?
+        @hare_info && @hare_info.connection && @hare_info.connection.open?
+      end
+
+      def connected?
+        return nil unless @hare_info && @hare_info.connection
+        @hare_info.connection.connected?
+      end
+
+      private
+
+      def declare_exchange!(channel, exchange, exchange_type, durable)
+        @logger.debug? && @logger.debug("Declaring an exchange", :name => exchange,
+                      :type => exchange_type, :durable => durable)
+        exchange = channel.exchange(exchange, :type => exchange_type.to_sym, :durable => durable)
+        @logger.debug? && @logger.debug("Exchange declared")
+        exchange
+      rescue StandardError => e
+        @logger.error("Could not declare exchange!",
+                      :exchange => exchange, :type => exchange_type,
+                      :durable => durable, :error_class => e.class.name,
+                      :error_message => e.message, :backtrace => e.backtrace)
+        raise e
+      end
+
+      def connect
+        @logger.debug? && @logger.debug("Connecting to RabbitMQ. Settings: #{rabbitmq_settings.inspect}")
+
+        connection = MarchHare.connect(rabbitmq_settings)
+
+        connection.on_shutdown do |conn, cause|
+           @logger.warn("RabbitMQ connection was closed!",
+                          :url => connection_url(conn),
+                          :automatic_recovery => @automatic_recovery,
+                          :cause => cause)
+        end
+        connection.on_blocked do
+          @logger.warn("RabbitMQ connection blocked! Check your RabbitMQ instance!",
+                       :url => connection_url(connection))
+        end
+        connection.on_unblocked do
+          @logger.warn("RabbitMQ connection unblocked!", :url => connection_url(connection))
+        end
+
+        channel = connection.create_channel
+        @logger.info("Connected to RabbitMQ at #{rabbitmq_settings[:host]}")
+
+        HareInfo.new(connection, channel)
+      end
+
+      # Mostly used for printing debug logs
+      def connection_url(connection)
+        user_pass = connection.user ? "#{connection.user}:XXXXXX@" : ""
+        protocol = params["ssl"] ? "amqps" : "amqp"
+        "#{protocol}://#{user_pass}#{connection.host}:#{connection.port}#{connection.vhost}"
+      end
+
+      def sleep_for_retry
+        Stud.stoppable_sleep(@connect_retry_interval) { @rabbitmq_connection_stopping }
+      end
+    end
+  end
+end

data/lib/logstash_registry.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+require "logstash/plugins/registry"
+require "logstash/inputs/rabbitmq"
+require "logstash/outputs/rabbitmq"
+
+LogStash::PLUGIN_REGISTRY.add(:input, "rabbitmq", LogStash::Inputs::RabbitMQ)
+LogStash::PLUGIN_REGISTRY.add(:output, "rabbitmq", LogStash::Outputs::RabbitMQ)

data/logstash-integration-rabbitmq.gemspec

@@ -0,0 +1,53 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-integration-rabbitmq'
+  s.version         = '7.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Integration with RabbitMQ - input and output plugins"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
+                      "using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = Dir.glob(%w(
+    lib/**/*
+    spec/**/*
+    *.gemspec
+    *.md
+    CONTRIBUTORS
+    Gemfile
+    LICENSE
+    NOTICE.TXT
+    vendor/jar-dependencies/**/*.jar
+    vendor/jar-dependencies/**/*.rb
+    VERSION docs/**/*
+  ))
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {
+      "logstash_plugin"     => "true",
+      "logstash_group"      => "integration",
+      "integration_plugins" => "logstash-input-rabbitmq,logstash-output-rabbitmq"
+  }
+
+  s.platform = RUBY_PLATFORM
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-core", ">= 6.5.0"
+
+  s.add_runtime_dependency 'logstash-codec-json'
+
+  s.add_runtime_dependency 'march_hare', ['~> 4.0'] #(MIT license)
+  s.add_runtime_dependency 'stud', '~> 0.0.22'
+  s.add_runtime_dependency 'back_pressure', '~> 1.0'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'logstash-codec-plain'
+end

data/spec/fixtures/README.md

@@ -0,0 +1,150 @@
+# TLS Testing notes
+
+Currently all TLS integration style testing is manual. This fixture may be used to help test. 
+
+* client - has the key store with client public/private key and the server public key
+* server - has the key store with server public/private key and the client public key
+* client_untrusted - has a keystore with public/private key that is self signed
+
+logstash config
+---------
+```
+input {
+  rabbitmq {
+    host => "localhost"
+    port => 5671
+    queue => "hello"
+    codec => plain
+    ssl => true
+    ssl_certificate_path => "/Users/jake/workspace/plugins/logstash-mixin-rabbitmq_connection/spec/fixtures/client/keycert.p12"
+    ssl_certificate_password => "MySecretPassword"
+  }
+}
+
+output{ stdout { codec => rubydebug } }
+```
+
+rabbit mq install
+--------
+(mac) 
+
+```
+brew install rabbitmq
+export PATH=$PATH:/usr/local/sbin
+vim /usr/local/etc/rabbitmq/rabbitmq.config
+
+```
+```
+[
+  {rabbit, [
+     {ssl_listeners, [5671]},
+     {ssl_options, [{cacertfile,"/Users/jake/workspace/plugins/logstash-mixin-rabbitmq_connection/spec/fixtures/testca/cacert.pem"},
+                    {certfile,"/Users/jake/workspace/plugins/logstash-mixin-rabbitmq_connection/spec/fixtures/server/cert.pem"},
+                    {keyfile,"/Users/jake/workspace/plugins/logstash-mixin-rabbitmq_connection/spec/fixtures/server/key.pem"},
+                    {verify,verify_peer},
+                    {fail_if_no_peer_cert,false}]}
+   ]}
+].
+```
+```
+export PATH=$PATH:/usr/local/sbin
+rabbitmq-server
+tail -f /usr/local/var/log/rabbitmq/rabbit@localhost.log
+```
+
+sending a test message with ruby
+----------
+https://www.rabbitmq.com/tutorials/tutorial-one-ruby.html
+
+```
+gem install bunny --version ">= 2.6.4"
+```
+Create a file called send.rb
+```
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require "bunny"
+
+conn = Bunny.new(:automatically_recover => false)
+conn.start
+
+ch   = conn.create_channel
+q    = ch.queue("hello")
+
+ch.default_exchange.publish("Test message", :routing_key => q.name)
+puts "Message sent'"
+
+conn.close
+```
+Send the message with Logstash running as the consumer
+``` 
+ruby send.rb 
+```
+
+password for all files
+--------
+MySecretPassword
+
+start from testca dir
+---------
+```
+cd testca
+```
+
+client
+-------
+```
+openssl genrsa -out ../client/key.pem 2048
+openssl req -config openssl.cnf  -key ../client/key.pem  -new -sha256 -out ../client/req.pem
+# for hostname use localhost, O=client
+openssl ca -config openssl.cnf  -in ../client/req.pem -out ../client/cert.pem
+openssl x509 -in ../client/cert.pem -text -noout
+openssl pkcs12 -export -out ../client/keycert.p12 -inkey ../client/key.pem -in ../client/cert.pem
+```
+
+server
+-------
+```
+openssl genrsa -out ../server/key.pem 2048
+openssl req -config openssl.cnf  -key ../server/key.pem  -new -sha256 -out ../server/req.pem
+# for hostname use localhost, O=server
+openssl ca -config openssl.cnf  -in ../server/req.pem -out ../server/cert.pem
+openssl x509 -in ../server/cert.pem -text -noout
+openssl pkcs12 -export -out ../server/keycert.p12 -inkey ../server/key.pem -in ../server/cert.pem
+```
+
+establish trust
+----------
+```
+cd server
+keytool -import -file ../client/cert.pem  -alias client_cert -keystore keycert.p12
+cd client
+keytool -import -file ../server/cert.pem  -alias server_cert -keystore keycert.p12
+```
+
+reading
+-----------
+```
+openssl x509 -in cert.pem -text -noout
+keytool -list -v -keystore keycert.p12
+```
+
+self signed cert (untrusted)
+-------
+ ```
+openssl req -x509  -batch -nodes -newkey rsa:2048 -keyout key.pem -out cert.pem -subj "/CN=localhost, O=client" -days 10000
+openssl pkcs12 -export -out keycert.p12 -inkey key.pem -in cert.pem
+
+```
+
+Issue [44](https://github.com/logstash-plugins/logstash-mixin-rabbitmq_connection/issues/44) validation
+---------
+configure Logstash to the untrusted cert :`ssl_certificate_path => "/Users/jake/workspace/plugins/logstash-mixin-rabbitmq_connection/spec/fixtures/client_untrusted/keycert.p12"`
+
+This _SHOULD_ fail an error like the following:
+```
+Using TLS/SSL version TLSv1.2
+[2017-12-19T12:47:12,891][ERROR][logstash.inputs.rabbitmq ] RabbitMQ connection error, will retry. {:error_message=>"sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target", :exception=>"Java::JavaxNetSsl::SSLHandshakeException"}
+```
+There should _NOT_ be any warnings about a NullTrustManager or disabling verification, and you should _NOT_ be able to send messages.