logstash-output-rabbitmq 0.1.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3d077636bc3938cec6ff586816627df8672759bc
+  data.tar.gz: 0e17b4216d83ef4478357cdc390ae930d2ab8b51
+SHA512:
+  metadata.gz: ad4a194dbab8de29976d76b7c8c509941382b28693ef825a86fbedfbe4fb852094ba5be40ae16590f9472d62eea9fde908080946504a43d125cab2bd7c95ae27
+  data.tar.gz: 9213a4f73ae57cd8e80899a51cc6ab0140f872c8ee0e642c29e3b4004b1d5cc95dc368b850fd8b1b3865c1508ccdf81eeb35f82390ad2c5cda82ddbd577baaa6

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/outputs/rabbitmq.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
+# or later version (3.x is recommended).
+#
+# Relevant links:
+#
+# * http://www.rabbitmq.com/[RabbitMQ]
+# * http://rubymarchhare.info[March Hare]
+# * http://rubybunny.info[Bunny]
+class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
+  EXCHANGE_TYPES = ["fanout", "direct", "topic"]
+
+  config_name "rabbitmq"
+  milestone 1
+
+
+  #
+  # Connection
+  #
+
+  # RabbitMQ server address
+  config :host, :validate => :string, :required => true
+
+  # RabbitMQ port to connect on
+  config :port, :validate => :number, :default => 5672
+
+  # RabbitMQ username
+  config :user, :validate => :string, :default => "guest"
+
+  # RabbitMQ password
+  config :password, :validate => :password, :default => "guest"
+
+  # The vhost to use. If you don't know what this is, leave the default.
+  config :vhost, :validate => :string, :default => "/"
+
+  # Enable or disable SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # Validate SSL certificate
+  config :verify_ssl, :validate => :boolean, :default => false
+
+  # Enable or disable logging
+  config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
+
+
+
+  #
+  # Exchange
+  #
+
+
+  # The exchange type (fanout, topic, direct)
+  config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
+
+  # The name of the exchange
+  config :exchange, :validate => :string, :required => true
+
+  # Key to route to by default. Defaults to 'logstash'
+  #
+  # * Routing keys are ignored on fanout exchanges.
+  config :key, :validate => :string, :default => "logstash"
+
+  # Is this exchange durable? (aka; Should it survive a broker restart?)
+  config :durable, :validate => :boolean, :default => true
+
+  # Should RabbitMQ persist messages to disk?
+  config :persistent, :validate => :boolean, :default => true
+
+
+
+  def initialize(params)
+    params["codec"] = "json" if !params["codec"]
+
+    super
+  end
+
+  # Use MarchHare on JRuby to avoid IO#select CPU spikes
+  # (see github.com/ruby-amqp/bunny/issues/95).
+  #
+  # On MRI, use Bunny.
+  #
+  # See http://rubybunny.info and http://rubymarchhare.info
+  # for the docs.
+  if RUBY_ENGINE == "jruby"
+    require "logstash/outputs/rabbitmq/march_hare"
+
+    include MarchHareImpl
+  else
+    require "logstash/outputs/rabbitmq/bunny"
+
+    include BunnyImpl
+  end
+end # class LogStash::Outputs::RabbitMQ

data/lib/logstash/outputs/rabbitmq/bunny.rb

@@ -0,0 +1,138 @@
+# encoding: utf-8
+
+require "logstash/json"
+
+class LogStash::Outputs::RabbitMQ
+  module BunnyImpl
+
+    #
+    # API
+    #
+
+    def register
+      require "bunny"
+
+      @logger.info("Registering output", :plugin => self)
+
+      connect
+      declare_exchange
+    end # def register
+
+
+    def receive(event)
+      return unless output?(event)
+
+      @logger.debug("Sending event", :destination => to_s, :event => event, :key => key)
+      key = event.sprintf(@key) if @key
+
+      begin
+        publish_serialized(event.to_json, key)
+      rescue LogStash::Json::GeneratorError => e
+        @logger.warn("Trouble converting event to JSON", :exception => e,
+                     :event => event)
+      end
+    end
+
+    def publish_serialized(message, key = @key)
+      begin
+        if @x
+          @x.publish(message, :persistent => @persistent, :routing_key => key)
+        else
+          @logger.warn("Tried to send a message, but not connected to RabbitMQ yet.")
+        end
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        connect
+        declare_exchange
+        retry
+      end
+    end
+
+    def to_s
+      return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
+    end
+
+    def teardown
+      @conn.close if @conn && @conn.open?
+      @conn = nil
+
+      finished
+    end
+
+
+
+    #
+    # Implementation
+    #
+
+    def connect
+      @vhost       ||= Bunny::DEFAULT_HOST
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= AMQ::Protocol::DEFAULT_PORT
+      @routing_key ||= "#"
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :automatically_recover => false
+      }
+      @settings[:user]      = @user || Bunny::DEFAULT_USER
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                Bunny::DEFAULT_PASSWORD
+                              end
+
+      @settings[:log_level] = if @debug || @logger.debug?
+                                :debug
+                              else
+                                :error
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      @settings[:verify_ssl] = @verify_ssl if @verify_ssl
+
+      proto                  = if @ssl
+                                 "amqp"
+                               else
+                                 "amqps"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      begin
+        @conn = Bunny.new(@settings)
+
+        @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+        return if terminating?
+        @conn.start
+
+        @ch = @conn.create_channel
+        @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+      rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
+        n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        retry
+      end
+    end
+
+    def declare_exchange
+      @logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
+                    :durable => @durable)
+      @x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
+    end
+  end # BunnyImpl
+end # LogStash::Outputs::RabbitMQ

data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb

@@ -0,0 +1 @@
+require "logstash/outputs/rabbitmq/march_hare"

data/lib/logstash/outputs/rabbitmq/march_hare.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+class LogStash::Outputs::RabbitMQ
+  module MarchHareImpl
+
+
+    #
+    # API
+    #
+
+    def register
+      require "march_hare"
+      require "java"
+
+      @logger.info("Registering output", :plugin => self)
+
+      @connected = java.util.concurrent.atomic.AtomicBoolean.new
+
+      connect
+      declare_exchange
+
+      @connected.set(true)
+
+      @codec.on_event(&method(:publish_serialized))
+    end
+
+
+    def receive(event)
+      return unless output?(event)
+
+      begin
+        @codec.encode(event)
+      rescue JSON::GeneratorError => e
+        @logger.warn("Trouble converting event to JSON", :exception => e,
+                     :event => event)
+      end
+    end
+
+    def publish_serialized(message)
+      begin
+        if @connected.get
+          @x.publish(message, :routing_key => @key, :properties => {
+            :persistent => @persistent
+          })
+        else
+          @logger.warn("Tried to send a message, but not connected to RabbitMQ.")
+        end
+      rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
+        @connected.set(false)
+        n = 10
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+
+        connect
+        declare_exchange
+        retry
+      end
+    end
+
+    def to_s
+      return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
+    end
+
+    def teardown
+      @connected.set(false)
+      @conn.close if @conn && @conn.open?
+      @conn = nil
+
+      finished
+    end
+
+
+
+    #
+    # Implementation
+    #
+
+    def connect
+      return if terminating?
+
+      @vhost       ||= "127.0.0.1"
+      # 5672. Will be switched to 5671 by Bunny if TLS is enabled.
+      @port        ||= 5672
+
+      @settings = {
+        :vhost => @vhost,
+        :host  => @host,
+        :port  => @port,
+        :user  => @user,
+        :automatic_recovery => false
+      }
+      @settings[:pass]      = if @password
+                                @password.value
+                              else
+                                "guest"
+                              end
+
+      @settings[:tls]        = @ssl if @ssl
+      proto                  = if @ssl
+                                 "amqp"
+                               else
+                                 "amqps"
+                               end
+      @connection_url        = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
+
+      begin
+        @conn = MarchHare.connect(@settings)
+
+        @logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
+
+        @ch = @conn.create_channel
+        @logger.info("Connected to RabbitMQ at #{@settings[:host]}")
+      rescue MarchHare::Exception => e
+        @connected.set(false)
+        n = 10
+
+        @logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
+                      :exception => e,
+                      :backtrace => e.backtrace)
+        return if terminating?
+
+        sleep n
+        retry
+      end
+    end
+
+    def declare_exchange
+      @logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
+                    :durable => @durable)
+      @x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
+
+      # sets @connected to true during recovery. MK.
+      @connected.set(true)
+
+      @x
+    end
+
+  end # MarchHareImpl
+end

data/logstash-output-rabbitmq.gemspec

@@ -0,0 +1,34 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-rabbitmq'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Push events to a RabbitMQ exchange"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+
+  if RUBY_PLATFORM == 'java'
+    s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency 'march_hare', ['~> 2.5.1'] #(MIT license)
+  else
+    s.add_runtime_dependency 'bunny', ['>= 1.6.0'] #(MIT license)
+  end
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/outputs/rabbitmq_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-rabbitmq
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: java
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  name: march_hare
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/outputs/rabbitmq.rb
+- lib/logstash/outputs/rabbitmq/bunny.rb
+- lib/logstash/outputs/rabbitmq/hot_bunnies.rb
+- lib/logstash/outputs/rabbitmq/march_hare.rb
+- logstash-output-rabbitmq.gemspec
+- spec/outputs/rabbitmq_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Push events to a RabbitMQ exchange
+test_files:
+- spec/outputs/rabbitmq_spec.rb