logstash-output-rabbitmq 1.1.2-java → 3.0.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12325a92dfaa36797e082e83812814298a74852e
-  data.tar.gz: 84ea55bb9a4d8edf5a1cfdef9d8212a61d2f058b
+  metadata.gz: 89f6e5f8795784e82af0619ddaff8931f40895ec
+  data.tar.gz: 952f9b827d7b5612fa5729e0ccef00e6d93a5076
 SHA512:
-  metadata.gz: 5112897974000d4fed166cd3e6f4238015dc4f9b7bd93fc7b98f6dd5b820b7301cb1f4a7e176ad0e3e0d1823b1d7e6c51d633b12105be5cb9887bd2ff89af748
-  data.tar.gz: 3ee72b113e4b87cbbed11388e5c57a9b59403e0cef1a3a12ed3e575541a365b2b4cfa0c5aa642c30623c364ddd32e5d8f4b8bd03615e5e1ae46ede41517f379c
+  metadata.gz: cb144570bebf2736e084cbc29002e3edd971f9a3b1f930b3292d342b3f277e53c91b7ef29827e7c9424e84590cb9d8684b7c7d480ad80bbd9969a2991f133316
+  data.tar.gz: cd0aa6c48edb51096ac223028e8e9c6ebc6c17a10b94e921639596e4feb1a3e0460346787b55424887f37aa566088fe0589dfcadc0605ca4a1494d1bc07c055c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+* 2.0.0
+  - Massive refactor
+  - Implement Logstash 2.x stop behavior
+  - Depend on rabbitmq_connection mixin for most connection functionality
 * 1.1.2
  - Bump march_hare to 2.12.0 to fix jar file permissions
 * 1.1.1

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/README.md

@@ -1,86 +1,4 @@
-# Logstash Plugin
+# About this plugin 
 
-This is a plugin for [Logstash](https://github.com/elastic/logstash).
-
-It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
-
-## Documentation
-
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
-
-- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
-
-## Need Help?
-
-Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
-
-## Developing
-
-### 1. Plugin Developement and Testing
-
-#### Code
-- To get started, you'll need JRuby with the Bundler gem installed.
-
-- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
-
-- Install dependencies
-```sh
-bundle install
-```
-
-#### Test
-
-- Update your dependencies
-
-```sh
-bundle install
-```
-
-- Run tests
-
-```sh
-bundle exec rspec
-```
-
-### 2. Running your unpublished Plugin in Logstash
-
-#### 2.1 Run in a local Logstash clone
-
-- Edit Logstash `Gemfile` and add the local plugin path, for example:
-```ruby
-gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
-```
-- Install plugin
-```sh
-bin/plugin install --no-verify
-```
-- Run Logstash with your plugin
-```sh
-bin/logstash -e 'filter {awesome {}}'
-```
-At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
-
-#### 2.2 Run in an installed Logstash
-
-You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
-
-- Build your plugin gem
-```sh
-gem build logstash-filter-awesome.gemspec
-```
-- Install the plugin from the Logstash home
-```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
-```
-- Start Logstash and proceed to test the plugin
-
-## Contributing
-
-All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
-
-Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
-
-It is more important to the community that you are able to contribute.
-
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+This is a mixin supporting  the logstash rabbitmq inputs and outputs. If you plan on writing a logstash plugin using
+rabbitmq this may be useful.

data/lib/logstash/outputs/rabbitmq.rb

@@ -1,8 +1,6 @@
-# encoding: utf-8
-require "logstash/outputs/base"
-require "logstash/namespace"
-require "march_hare"
-require "java"
+# encoding: UTF-8
+require "logstash/pipeline"
+require "logstash/plugin_mixins/rabbitmq_connection"
 
 # Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
 # or later version (3.x is recommended).
@@ -11,178 +9,58 @@ require "java"
 #
 # * http://www.rabbitmq.com/[RabbitMQ]
 # * http://rubymarchhare.info[March Hare]
-class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
-  EXCHANGE_TYPES = ["fanout", "direct", "topic"]
-
-  HareInfo = Struct.new(:connection, :channel, :exchange)
-
-  config_name "rabbitmq"
-
-  # RabbitMQ server address
-  config :host, :validate => :string, :required => true
-
-  # RabbitMQ port to connect on
-  config :port, :validate => :number, :default => 5672
-
-  # RabbitMQ username
-  config :user, :validate => :string, :default => "guest"
-
-  # RabbitMQ password
-  config :password, :validate => :password, :default => "guest"
-
-  # The vhost to use. If you don't know what this is, leave the default.
-  config :vhost, :validate => :string, :default => "/"
-
-  # Enable or disable SSL
-  config :ssl, :validate => :boolean, :default => false
-
-  # Validate SSL certificate
-  config :verify_ssl, :validate => :boolean, :default => false
-
-  # Enable or disable logging
-  config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
-
-  # Set this to automatically recover from a broken connection. You almost certainly don't want to override this!!!
-  config :automatic_recovery, :validate => :boolean, :default => true
-
-  # The exchange type (fanout, topic, direct)
-  config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
-
-  # The name of the exchange
-  config :exchange, :validate => :string, :required => true
-
-  # Key to route to by default. Defaults to 'logstash'
-  #
-  # * Routing keys are ignored on fanout exchanges.
-  config :key, :validate => :string, :default => "logstash"
-
-  # Is this exchange durable? (aka; Should it survive a broker restart?)
-  config :durable, :validate => :boolean, :default => true
-
-  # Should RabbitMQ persist messages to disk?
-  config :persistent, :validate => :boolean, :default => true
-
-  # Time in seconds to wait before retrying a connection
-  config :connect_retry_interval, :validate => :number, :default => 1
-
-  def initialize(params)
-    params["codec"] = "json" if !params["codec"]
-
-    super
-  end
-
-  public
-  def register
-    connect!
-    @codec.on_event(&method(:publish))
-  end
-
-  public
-  def receive(event)
-    return unless output?(event)
-
-    @codec.encode(event)
-  rescue StandardError => e
-    @logger.warn("Error encoding event", :exception => e, :event => event)
-  end
-
-  private
-  def publish(event, message)
-    @hare_info.exchange.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
-  rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
-    return if terminating?
-
-    @logger.error("Error while publishing. Will retry.",
-                  :message => e.message,
-                  :exception => e.class,
-                  :backtrace => e.backtrace)
-
-    sleep_for_retry
-    retry
-  end
-
-  public
-  def to_s
-    return "<LogStash::RabbitMQ::Output: amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}>"
-  end
-
-  public
-  def teardown
-    @hare_info.connection.close if connection_open?
-
-    finished
-  end
-
-  private
-  def settings
-    return @settings if @settings
-
-    s = {
-      :vhost => @vhost,
-      :host  => @host,
-      :port  => @port,
-      :user  => @user,
-      :automatic_recovery => @automatic_recovery,
-      :pass => @password ? @password.value : "guest",
-    }
-    s[:tls] = @ssl if @ssl
-    @settings = s
-  end
-
-  private
-  def connect
-    @logger.debug("Connecting to RabbitMQ. Settings: #{settings.inspect}, queue: #{@queue.inspect}")
-
-
-    connection = MarchHare.connect(settings)
-    connection.on_blocked { @logger.warn("RabbitMQ output blocked! Check your RabbitMQ instance!") }
-    connection.on_unblocked { @logger.warn("RabbitMQ output unblocked!") }
-
-    channel = connection.create_channel
-    @logger.info("Connected to RabbitMQ at #{settings[:host]}")
-
-    exchange = declare_exchange!(channel)
-
-    HareInfo.new(connection, channel, exchange)
-  end
-
-  private
-  def declare_exchange!(channel)
-    @logger.debug("Declaring an exchange", :name => @exchange,
-                  :type => @exchange_type, :durable => @durable)
-    exchange = channel.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
-    @logger.debug("Exchange declared")
-    exchange
-  rescue StandardError => e
-    @logger.error("Could not declare exchange!",
-                  :exchange => @exchange, :type => @exchange_type,
-                  :durable => @durable, :error_class => e.class.name,
-                  :error_message => e.message, :backtrace => e.backtrace)
-    raise e
-  end
-
-  private
-  def connect!
-    @hare_info = connect()
-  rescue MarchHare::Exception => e
-    return if terminating?
-
-    @logger.error("RabbitMQ connection error, will retry.",
-                  :message => e.message,
-                  :exception => e.class.name,
-                  :backtrace => e.backtrace)
-
-    sleep_for_retry
-    retry
-  end
-
-  private
-  def connection_open?
-    @hare_info && @hare_info.connection && @hare_info.connection.open?
-  end
-
-  private
-  def sleep_for_retry
-    sleep @connect_retry_interval
+module LogStash
+  module Outputs
+    class RabbitMQ < LogStash::Outputs::Base
+      include LogStash::PluginMixins::RabbitMQConnection
+
+      # Key to route to by default. Defaults to 'logstash'
+      #
+      # * Routing keys are ignored on fanout exchanges.
+      config :key, :validate => :string, :default => "logstash"
+
+      # The name of the exchange
+      config :exchange, :validate => :string, :required => true
+
+      # The exchange type (fanout, topic, direct)
+      config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
+
+      # Is this exchange durable? (aka; Should it survive a broker restart?)
+      config :durable, :validate => :boolean, :default => true
+
+      # Should RabbitMQ persist messages to disk?
+      config :persistent, :validate => :boolean, :default => true
+
+      def register
+        connect!
+        @hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
+        @codec.on_event(&method(:publish))
+      end
+
+      def receive(event)
+        return unless output?(event)
+
+        @codec.encode(event)
+      rescue StandardError => e
+        @logger.warn("Error encoding event", :exception => e, :event => event)
+      end
+
+      def publish(event, message)
+        raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
+        @hare_info.exchange.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
+      rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
+        @logger.error("Error while publishing. Will retry.",
+                      :message => e.message,
+                      :exception => e.class,
+                      :backtrace => e.backtrace)
+
+        sleep_for_retry
+        retry
+      end
+
+      def close
+        close_connection
+      end
+    end
   end
 end

data/logstash-output-rabbitmq.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-rabbitmq'
-  s.version         = '1.1.2'
+  s.version         = '3.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Push events to a RabbitMQ exchange"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,10 +20,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
+  s.add_runtime_dependency "logstash-mixin-rabbitmq_connection", '>= 1.0.0', '< 2.0.0'
 
   s.platform = RUBY_PLATFORM
-  s.add_runtime_dependency 'march_hare', ['~> 2.12.0'] #(MIT license)
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-input-generator'

data/spec/outputs/rabbitmq_spec.rb

@@ -1,5 +1,5 @@
+# encoding: UTF-8
 require "logstash/devutils/rspec/spec_helper"
-
 require "logstash/pipeline"
 require "logstash/outputs/rabbitmq"
 
@@ -11,7 +11,7 @@ describe LogStash::Outputs::RabbitMQ do
   let(:exchange) { "myexchange" }
   let(:key) { "mykey" }
   let(:persistent) { true }
-  let(:settings) {
+  let(:rabbitmq_settings) {
     {
       "host" => host,
       "port" => port,
@@ -21,7 +21,7 @@ describe LogStash::Outputs::RabbitMQ do
       "persistent" => persistent
     }
   }
-  let(:instance) { klass.new(settings) }
+  let(:instance) { klass.new(rabbitmq_settings) }
   let(:hare_info) { instance.instance_variable_get(:@hare_info) }
 
   context "when connected" do
@@ -40,33 +40,9 @@ describe LogStash::Outputs::RabbitMQ do
       instance.register
     end
 
-    describe "#register" do
-      subject { instance }
-
-      it "should create cleanly" do
-        expect(subject).to be_a(klass)
-      end
-
-      it "should connect" do
-        expect(subject).to have_received(:connect!).once
-      end
-    end
-
     describe "#connect!" do
       subject { hare_info }
 
-      it "should set @hare_info correctly" do
-        expect(subject).to be_a(LogStash::Outputs::RabbitMQ::HareInfo)
-      end
-
-      it "should set @connection correctly" do
-        expect(subject.connection).to eql(connection)
-      end
-
-      it "should set the channel correctly" do
-        expect(subject.channel).to eql(channel)
-      end
-
       it "should set the exchange correctly" do
         expect(subject.exchange).to eql(exchange)
       end
@@ -128,36 +104,86 @@ describe LogStash::Outputs::RabbitMQ do
       end
     end
   end
+end
 
-  # If the connection encounters an exception during its initial
-  # connection attempt we must handle that. Subsequent errors should be
-  # handled by the automatic retry mechanism built-in to MarchHare
-  describe "initial connection exceptions" do
-    subject { instance }
 
-    before do
-      allow(subject).to receive(:sleep_for_retry)
+describe "with a live server", :integration => true do
+  let(:klass) { LogStash::Outputs::RabbitMQ }
+  let(:exchange) { "myexchange" }
+  let(:exchange_type) { "topic" }
+  let(:default_plugin_config) {
+    {
+      "host" => "127.0.0.1",
+      "exchange" => exchange,
+      "exchange_type" => exchange_type,
+      "key" => "foo"
+    }
+  }
+  let(:config) { default_plugin_config }
+  let(:instance) { klass.new(config) }
+  let(:hare_info) { instance.instance_variable_get(:@hare_info) }
 
+  # Spawn a connection in the bg and wait up (n) seconds
+  def spawn_and_wait(instance)
+    instance.register
 
-      i = 0
-      allow(subject).to receive(:connect) do
-        i += 1
-        if i == 1
-          raise(MarchHare::ConnectionRefused, "Error!")
-        else
-          double("connection")
-        end
-      end
+    20.times do
+      instance.connected? ? break : sleep(0.1)
+    end
 
-      subject.send(:connect!)
+    # Extra time to make sure the output can attach
+    sleep 1
+  end
+
+  let(:test_connection) { MarchHare.connect(instance.send(:rabbitmq_settings)) }
+  let(:test_channel) { test_connection.create_channel }
+  let(:test_queue) {
+    test_channel.queue("testq", :auto_delete => true).bind(exchange, :key => config["key"])
+  }
+
+  before do
+    # Materialize the instance in the current thread to prevent dupes
+    # If you use multiple threads with lazy evaluation weird stuff happens
+    instance
+    spawn_and_wait(instance)
+
+    test_channel # Start up the test client as well
+    test_queue
+  end
+
+  after do
+    instance.close()
+    test_channel.close
+    test_connection.close
+  end
+
+  context "using defaults" do
+    it "should start, connect, and stop cleanly" do
+      expect(instance.connected?).to be_truthy
     end
 
-    it "should retry its connection when conn fails" do
-      expect(subject).to have_received(:connect).twice
+    it "should close cleanly" do
+      instance.close
+      expect(instance.connected?).to be_falsey
     end
+  end
+
+  describe "sending a message with an exchange specified" do
+    let(:message) { LogStash::Event.new(:message => "Foo Message") }
+
+    it "should process the message" do
+      @received = nil
+      test_queue.subscribe do |metadata,payload|
+        @received = payload
+      end
+
+      instance.receive(message)
+
+      until @received
+        sleep 1
+      end
 
-    it "should sleep between retries" do
-      expect(subject).to have_received(:sleep_for_retry).once
+      expect(@received).to eql(message.to_s)
     end
   end
 end

metadata

@@ -1,47 +1,47 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-rabbitmq
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 3.0.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-27 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: march_hare
+  name: logstash-mixin-rabbitmq_connection
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - <
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 1.0.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -92,18 +92,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- lib/logstash/outputs/rabbitmq.rb
+- spec/outputs/rabbitmq_spec.rb
+- logstash-output-rabbitmq.gemspec
 - CHANGELOG.md
+- README.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
-- README.md
-- Rakefile
-- examples/sample.conf
-- lib/logstash/outputs/rabbitmq.rb
-- logstash-output-rabbitmq.gemspec
-- spec/outputs/rabbitmq_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"

data/examples/sample.conf

@@ -1,11 +0,0 @@
-input {
-  stdin {}
-}
-
-output {
-  rabbitmq {
-    host => "127.0.0.1"
-    exchange => "foo"
-    exchange_type => "topic"
-  }
-}