logstash-integration-rabbitmq 0.0.1

data/docs/index-output.asciidoc

@@ -0,0 +1,266 @@
+:plugin: rabbitmq
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Rabbitmq output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
+or later version (3.x is recommended).
+
+Relevant links:
+
+* http://www.rabbitmq.com/[RabbitMQ]
+* http://rubymarchhare.info[March Hare]
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Rabbitmq Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-arguments>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-automatic_recovery>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-connect_retry_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-connection_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-durable>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-exchange>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-exchange_type>> |<<string,string>>, one of `["fanout", "direct", "topic", "x-consistent-hash", "x-modulus-hash"]`|Yes
+| <<plugins-{type}s-{plugin}-heartbeat>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-message_properties>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-passive>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-persistent>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_password>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_path>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ssl_version>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-vhost>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-arguments"]
+===== `arguments` 
+
+  * Value type is <<array,array>>
+  * Default value is `{}`
+
+Extra queue arguments as an array.
+To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
+
+[id="plugins-{type}s-{plugin}-automatic_recovery"]
+===== `automatic_recovery` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Set this to automatically recover from a broken connection. You almost certainly don't want to override this!!!
+
+[id="plugins-{type}s-{plugin}-connect_retry_interval"]
+===== `connect_retry_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Time in seconds to wait before retrying a connection
+
+[id="plugins-{type}s-{plugin}-connection_timeout"]
+===== `connection_timeout` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+The default connection timeout in milliseconds. If not specified the timeout is infinite.
+
+[id="plugins-{type}s-{plugin}-durable"]
+===== `durable` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Is this exchange durable? (aka; Should it survive a broker restart?)
+
+[id="plugins-{type}s-{plugin}-exchange"]
+===== `exchange` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the exchange
+
+[id="plugins-{type}s-{plugin}-exchange_type"]
+===== `exchange_type` 
+
+  * This is a required setting.
+  * Value can be any of: `fanout`, `direct`, `topic`, `x-consistent-hash`, `x-modulus-hash`
+  * There is no default value for this setting.
+
+The exchange type (fanout, topic, direct)
+
+[id="plugins-{type}s-{plugin}-heartbeat"]
+===== `heartbeat` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+Heartbeat delay in seconds. If unspecified no heartbeats will be sent
+
+[id="plugins-{type}s-{plugin}-host"]
+===== `host` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Common functionality for the rabbitmq input/output
+RabbitMQ server address(es)
+host can either be a single host, or a list of hosts
+i.e.
+  host => "localhost"
+or
+  host => ["host01", "host02]
+
+if multiple hosts are provided on the initial connection and any subsequent
+recovery attempts of the hosts is chosen at random and connected to.
+Note that only one host connection is active at a time.
+
+[id="plugins-{type}s-{plugin}-key"]
+===== `key` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+The default codec for this plugin is JSON. You can override this to suit your particular needs however.
+Key to route to by default. Defaults to 'logstash'
+
+* Routing keys are ignored on fanout exchanges.
+
+[id="plugins-{type}s-{plugin}-message_properties"]
+===== `message_properties`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Add properties to be set per-message here, such as 'Content-Type', 'Priority'
+
+Example:
+[source,ruby]
+    message_properties => { "priority" => "1" }
+
+
+[id="plugins-{type}s-{plugin}-passive"]
+===== `passive` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Passive queue creation? Useful for checking queue existance without modifying server state
+
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * Default value is `"guest"`
+
+RabbitMQ password
+
+[id="plugins-{type}s-{plugin}-persistent"]
+===== `persistent` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Should RabbitMQ persist messages to disk?
+
+[id="plugins-{type}s-{plugin}-port"]
+===== `port` 
+
+  * Value type is <<number,number>>
+  * Default value is `5672`
+
+RabbitMQ port to connect on
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl` 
+
+  * Value type is <<boolean,boolean>>
+  * There is no default value for this setting.
+
+Enable or disable SSL.
+Note that by default remote certificate verification is off.
+Specify ssl_certificate_path and ssl_certificate_password if you need
+certificate verification
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_password"]
+===== `ssl_certificate_password` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Password for the encrypted PKCS12 (.p12) certificate file specified in ssl_certificate_path
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_path"]
+===== `ssl_certificate_path` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+Path to an SSL certificate in PKCS12 (.p12) format used for verifying the remote host
+
+[id="plugins-{type}s-{plugin}-ssl_version"]
+===== `ssl_version` 
+
+  * Value type is <<string,string>>
+  * Default value is `"TLSv1.2"`
+
+Version of the SSL protocol to use.
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * Value type is <<string,string>>
+  * Default value is `"guest"`
+
+RabbitMQ username
+
+[id="plugins-{type}s-{plugin}-vhost"]
+===== `vhost` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/"`
+
+The vhost (virtual host) to use. If you don't know what this
+is, leave the default. With the exception of the default
+vhost ("/"), names of vhosts should not begin with a forward
+slash.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/rabbitmq.rb

@@ -0,0 +1,317 @@
+# encoding: utf-8
+require 'logstash/plugin_mixins/rabbitmq_connection'
+require 'logstash/inputs/threadable'
+require 'logstash/event'
+java_import java.util.concurrent.ArrayBlockingQueue
+java_import java.util.concurrent.TimeUnit
+
+module LogStash
+  module Inputs
+    # Pull events from a http://www.rabbitmq.com/[RabbitMQ] queue.
+    #
+    # The default settings will create an entirely transient queue and listen for all messages by default.
+    # If you need durability or any other advanced settings, please set the appropriate options
+    #
+    # This plugin uses the http://rubymarchhare.info/[March Hare] library
+    # for interacting with the RabbitMQ server. Most configuration options
+    # map directly to standard RabbitMQ and AMQP concepts. The
+    # https://www.rabbitmq.com/amqp-0-9-1-reference.html[AMQP 0-9-1 reference guide]
+    # and other parts of the RabbitMQ documentation are useful for deeper
+    # understanding.
+    #
+    # The properties of messages received will be stored in the
+    # `[@metadata][rabbitmq_properties]` field if the `@metadata_enabled` setting is checked.
+    # Note that storing metadata may degrade performance.
+    # The following properties may be available (in most cases dependent on whether
+    # they were set by the sender):
+    #
+    # * app-id
+    # * cluster-id
+    # * consumer-tag
+    # * content-encoding
+    # * content-type
+    # * correlation-id
+    # * delivery-mode
+    # * exchange
+    # * expiration
+    # * message-id
+    # * priority
+    # * redeliver
+    # * reply-to
+    # * routing-key
+    # * timestamp
+    # * type
+    # * user-id
+    #
+    # For example, to get the RabbitMQ message's timestamp property
+    # into the Logstash event's `@timestamp` field, use the date
+    # filter to parse the `[@metadata][rabbitmq_properties][timestamp]`
+    # field:
+    # [source,ruby]
+    #     filter {
+    #       if [@metadata][rabbitmq_properties][timestamp] {
+    #         date {
+    #           match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]
+    #         }
+    #       }
+    #     }
+    #
+    # Additionally, any message headers will be saved in the
+    # `[@metadata][rabbitmq_headers]` field.
+    class RabbitMQ < LogStash::Inputs::Threadable
+      include ::LogStash::PluginMixins::RabbitMQConnection
+
+      # The properties to extract from each message and store in a
+      # @metadata field.
+      #
+      # Technically the exchange, redeliver, and routing-key
+      # properties belong to the envelope and not the message but we
+      # ignore that distinction here. However, we extract the
+      # headers separately via get_headers even though the header
+      # table technically is a message property.
+      #
+      # Freezing all strings so that code modifying the event's
+      # @metadata field can't touch them.
+      #
+      # If updating this list, remember to update the documentation
+      # above too.
+      MESSAGE_PROPERTIES = [
+        "app-id",
+        "cluster-id",
+        "consumer-tag",
+        "content-encoding",
+        "content-type",
+        "correlation-id",
+        "delivery-mode",
+        "exchange",
+        "expiration",
+        "message-id",
+        "priority",
+        "redeliver",
+        "reply-to",
+        "routing-key",
+        "timestamp",
+        "type",
+        "user-id",
+      ].map { |s| s.freeze }.freeze
+
+      INTERNAL_QUEUE_POISON=[]
+
+      config_name "rabbitmq"
+
+      # The default codec for this plugin is JSON. You can override this to suit your particular needs however.
+      default :codec, "json"
+
+      # The name of the queue Logstash will consume events from. If
+      # left empty, a transient queue with an randomly chosen name
+      # will be created.
+      config :queue, :validate => :string, :default => ""
+
+      # Is this queue durable? (aka; Should it survive a broker restart?)
+      config :durable, :validate => :boolean, :default => false
+
+      # Should the queue be deleted on the broker when the last consumer
+      # disconnects? Set this option to `false` if you want the queue to remain
+      # on the broker, queueing up messages until a consumer comes along to
+      # consume them.
+      config :auto_delete, :validate => :boolean, :default => false
+
+      # Is the queue exclusive? Exclusive queues can only be used by the connection
+      # that declared them and will be deleted when it is closed (e.g. due to a Logstash
+      # restart).
+      config :exclusive, :validate => :boolean, :default => false
+
+      # Extra queue arguments as an array.
+      # To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
+      config :arguments, :validate => :array, :default => {}
+
+      # Prefetch count. If acknowledgements are enabled with the `ack`
+      # option, specifies the number of outstanding unacknowledged
+      # messages allowed.
+      config :prefetch_count, :validate => :number, :default => 256
+
+      # Enable message acknowledgements. With acknowledgements
+      # messages fetched by Logstash but not yet sent into the
+      # Logstash pipeline will be requeued by the server if Logstash
+      # shuts down. Acknowledgements will however hurt the message
+      # throughput.
+      #
+      # This will only send an ack back every `prefetch_count` messages.
+      # Working in batches provides a performance boost here.
+      config :ack, :validate => :boolean, :default => true
+
+      # If true the queue will be passively declared, meaning it must
+      # already exist on the server. To have Logstash create the queue
+      # if necessary leave this option as false. If actively declaring
+      # a queue that already exists, the queue options for this plugin
+      # (durable etc) must match those of the existing queue.
+      config :passive, :validate => :boolean, :default => false
+
+      # The name of the exchange to bind the queue to. Specify `exchange_type`
+      # as well to declare the exchange if it does not exist
+      config :exchange, :validate => :string
+
+      # The type of the exchange to bind to. Specifying this will cause this plugin
+      # to declare the exchange if it does not exist.
+      config :exchange_type, :validate => :string
+
+      # The routing key to use when binding a queue to the exchange.
+      # This is only relevant for direct or topic exchanges.
+      #
+      # * Routing keys are ignored on fanout exchanges.
+      # * Wildcards are not valid on direct exchanges.
+      config :key, :validate => :string, :default => "logstash"
+
+      # Amount of time in seconds to wait after a failed subscription request
+      # before retrying. Subscribes can fail if the server goes away and then comes back.
+      config :subscription_retry_interval_seconds, :validate => :number, :required => true, :default => 5
+
+      # Enable the storage of message headers and properties in `@metadata`. This may impact performance
+      config :metadata_enabled, :validate => :boolean, :default => false
+
+      def register
+        @internal_queue = java.util.concurrent.ArrayBlockingQueue.new(@prefetch_count*2)
+      end
+
+      def run(output_queue)
+        setup!
+        @output_queue = output_queue
+        consume!
+      end
+
+      def setup!
+        connect!
+        declare_queue!
+        bind_exchange!
+        @hare_info.channel.prefetch = @prefetch_count
+      rescue => e
+        @logger.warn("Error while setting up connection for rabbitmq input! Will retry.",
+                     :message => e.message,
+                     :class => e.class.name,
+                     :location => e.backtrace.first)
+        sleep_for_retry
+        retry
+      end
+
+      def bind_exchange!
+        if @exchange
+          if @exchange_type # Only declare the exchange if @exchange_type is set!
+            @logger.info? && @logger.info("Declaring exchange '#{@exchange}' with type #{@exchange_type}")
+            @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
+          end
+          @hare_info.queue.bind(@exchange, :routing_key => @key)
+        end
+      end
+
+      def declare_queue!
+        @hare_info.queue = declare_queue()
+      end
+
+      def declare_queue
+        @hare_info.channel.queue(@queue,
+                                 :durable     => @durable,
+                                 :auto_delete => @auto_delete,
+                                 :exclusive   => @exclusive,
+                                 :passive     => @passive,
+                                 :arguments   => @arguments)
+      end
+
+      def consume!
+        @consumer = @hare_info.queue.build_consumer(:on_cancellation => Proc.new { on_cancellation }) do |metadata, data|
+          @internal_queue.put [metadata, data]
+        end
+
+        begin
+          @hare_info.queue.subscribe_with(@consumer, :manual_ack => @ack)
+        rescue MarchHare::Exception => e
+          @logger.warn("Could not subscribe to queue! Will retry in #{@subscription_retry_interval_seconds} seconds", :queue => @queue)
+
+          sleep @subscription_retry_interval_seconds
+          retry
+        end
+
+        internal_queue_consume!
+      end
+
+      def internal_queue_consume!
+        i=0
+        last_delivery_tag=nil
+        while true
+          payload = @internal_queue.poll(10, TimeUnit::MILLISECONDS)
+          if !payload  # Nothing in the queue
+            if last_delivery_tag # And we have unacked stuff
+              @hare_info.channel.ack(last_delivery_tag, true) if @ack
+              i=0
+              last_delivery_tag = nil
+            end
+            next
+          end
+
+          break if payload == INTERNAL_QUEUE_POISON
+
+          metadata, data = payload
+          @codec.decode(data) do |event|
+            decorate(event)
+            if @metadata_enabled
+              event.set("[@metadata][rabbitmq_headers]", get_headers(metadata))
+              event.set("[@metadata][rabbitmq_properties]", get_properties(metadata))
+            end
+            @output_queue << event if event
+          end
+
+          i += 1
+
+          if i >= @prefetch_count
+            @hare_info.channel.ack(metadata.delivery_tag, true) if @ack
+            i = 0
+            last_delivery_tag = nil
+          else
+            last_delivery_tag = metadata.delivery_tag
+          end
+        end
+      end
+
+      def stop
+        @internal_queue.put(INTERNAL_QUEUE_POISON)
+        shutdown_consumer
+        close_connection
+      end
+
+      def shutdown_consumer
+        return unless @consumer
+        @hare_info.channel.basic_cancel(@consumer.consumer_tag)
+        until @consumer.terminated?
+          @logger.info("Waiting for rabbitmq consumer to terminate before stopping!", :params => self.params)
+          sleep 1
+        end
+      end
+
+      def on_cancellation
+        if !stop? # If this isn't already part of a regular stop
+          @logger.info("Received basic.cancel from #{rabbitmq_settings[:host]}, shutting down.")
+          stop
+        end
+      end
+
+      private
+      def get_headers(metadata)
+	metadata.headers || {}
+      end
+
+      private
+      def get_properties(metadata)
+        MESSAGE_PROPERTIES.reduce({}) do |acc, name|
+          # The method names obviously can't contain hyphens.
+          value = metadata.send(name.gsub("-", "_"))
+          if value
+            # The AMQP 0.9.1 timestamp field only has second resolution
+            # so storing milliseconds serves no purpose and might give
+            # the incorrect impression of a higher resolution.
+            acc[name] = name != "timestamp" ? value : value.getTime / 1000
+          end
+          acc
+        end
+      end
+    end
+  end
+end