hot_bunnies 1.3.8-java → 1.4.0-java

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 pkg/*
 .DS_Store
 .rvmrc
+bin/*

data/.travis.yml

@@ -7,3 +7,6 @@ rvm:
 notifications:
   recipients:
     - michaelklishin@me.com
+
+services:
+  - rabbitmq

data/ChangeLog.md

@@ -0,0 +1,33 @@
+# Changes Between 1.3.0 and 1.4.0
+
+## RabbitMQ Java Client Upgrade
+
+Hot Bunnies now uses RabbitMQ Java client 2.8.7.
+
+
+## TLS Support
+
+`HotBunnies.connect` now supports a new `:tls` option:
+
+``` ruby
+HotBunnies.connect(:tls => true)
+
+HotBunnies.connect(:tls => "SSLv3")
+HotBunnies.connect(:tls => "SSLv2")
+
+HotBunnies.connect(:tls => "SSLv3", :trust_manager => custom_trust_manager)
+```
+
+
+## Consumer Back Pressure Improvements
+
+  * The async consumer will not attempt to add tasks when its executor is shutting down.
+
+  * The blocking consumer got a buffer size option that makes it create a bounded blocking queue instead of an unbounded.
+
+
+## Consumer Improvements
+
+`HotBunnies::Queue#subscribe` is now more resilient to exceptions and uses a new
+executor task for each delivery. When a consumer is cancelled, any remaining messages
+will be delivered instead of being ignored.

data/README.md

@@ -25,6 +25,22 @@ Hot Bunnies is not
  * A cure for cancer
 
 
+## Installation, Dependency
+
+### With Rubygems
+
+    gem install hot_bunnies
+
+### With Bundler
+
+    gem "hot_bunnies", "~> 1.4.0"
+
+
+## Change Log
+
+See ChangeLog.md.
+
+
 ## Continuous Integration
 
 [![Continuous Integration status](https://secure.travis-ci.org/ruby-amqp/hot_bunnies.png)](http://travis-ci.org/ruby-amqp/hot_bunnies)
@@ -39,4 +55,4 @@ MIT, see LICENSE in the repository root
 
 ## Copyright
 
-Theo Hultberg, 2011.
+Theo Hultberg, Michael Klishin, 2011-2012.

data/lib/hot_bunnies/queue.rb

@@ -1,5 +1,15 @@
 # encoding: utf-8
 
+module JavaConcurrent
+  java_import 'java.lang.Thread'
+  java_import 'java.lang.InterruptedException'
+  java_import 'java.util.concurrent.Executors'
+  java_import 'java.util.concurrent.LinkedBlockingQueue'
+  java_import 'java.util.concurrent.ArrayBlockingQueue'
+  java_import 'java.util.concurrent.TimeUnit'
+  java_import 'java.util.concurrent.atomic.AtomicBoolean'
+end
+
 module HotBunnies
   class Queue
     attr_reader :name, :channel
@@ -59,6 +69,8 @@ module HotBunnies
     end
 
     class Subscription
+      include JavaConcurrent
+
       attr_reader :channel, :queue_name, :consumer_tag
 
       def initialize(channel, queue_name, options={})
@@ -66,28 +78,25 @@ module HotBunnies
         @queue_name = queue_name
         @ack        = options.fetch(:ack, false)
 
-        @cancelled  = java.util.concurrent.atomic.AtomicBoolean.new(false)
+        @cancelled  = AtomicBoolean.new(false)
       end
 
       def each(options={}, &block)
-        raise 'The subscription already has a message listener' if @subscriber
-        if options.fetch(:blocking, true)
-          run(&block)
-        else
-          if options[:executor]
-            @shut_down_executor = false
-            @executor = options[:executor]
-          else
-            @shut_down_executor = true
-            @executor = java.util.concurrent.Executors.new_single_thread_executor
-          end
-          @executor.submit { run(&block) }
-        end
+        raise 'The subscription already has a message listener' if @consumer
+        start(create_consumer(options, block))
+        nil
+      end
+      alias_method :each_message, :each
+
+      def start(consumer)
+        @consumer = consumer
+        @consumer_tag = @channel.basic_consume(@queue_name, !@ack, @consumer)
+        @consumer.start
       end
 
       def cancel
         raise 'Can\'t cancel: the subscriber haven\'t received an OK yet' if !self.active?
-        @channel.basic_cancel(@subscriber.consumer_tag)
+        @consumer.cancel
 
         # RabbitMQ Java client won't clear consumer_tag from cancelled consumers,
         # so we have to do this. Sharing consumers
@@ -103,7 +112,7 @@ module HotBunnies
       end
 
       def active?
-        !@cancelled.get && !@subscriber.nil? && !@subscriber.consumer_tag.nil?
+        !@cancelled.get && !@consumer.nil? && !@consumer.consumer_tag.nil?
       end
 
       def shutdown!
@@ -111,96 +120,198 @@ module HotBunnies
           @executor.shutdown_now
         end
       end
+      alias shut_down! shutdown!
 
       private
 
       def maybe_shutdown_executor
         if @executor && @shut_down_executor
           @executor.shutdown
+          unless @executor.await_termination(1, TimeUnit::SECONDS)
+            @executor.shutdown_now
+          end
         end
       end
 
-      def run(&block)
-        @subscriber = BlockingSubscriber.new(@channel, self)
-        @consumer_tag = @channel.basic_consume(@queue_name, !@ack, @subscriber.consumer)
-        @subscriber.on_message(&block)
+      def create_consumer(options, callback)
+        if options.fetch(:blocking, true)
+          BlockingCallbackConsumer.new(@channel, options[:buffer_size], callback)
+        else
+          if options[:executor]
+            @shut_down_executor = false
+            @executor = options[:executor]
+          else
+            @shut_down_executor = true
+            @executor = Executors.new_single_thread_executor
+          end
+          AsyncCallbackConsumer.new(@channel, callback, @executor)
+        end
       end
     end
 
-    class Headers
-      attr_reader :channel, :consumer_tag, :envelope, :properties
+    public
 
-      def initialize(channel, consumer_tag, envelope, properties)
-        @channel = channel
-        @consumer_tag = consumer_tag
-        @envelope = envelope
-        @properties = properties
+    class BaseConsumer < DefaultConsumer
+      def handleDelivery(consumer_tag, envelope, properties, body)
+        body = String.from_java_bytes(body)
+        headers = Headers.new(channel, consumer_tag, envelope, properties)
+        deliver(headers, body)
       end
 
-      def ack(options={})
-        @channel.basic_ack(delivery_tag, options.fetch(:multiple, false))
+      def handleCancel(consumer_tag)
+        @cancelled = true
       end
 
-      def reject(options={})
-        @channel.basic_reject(delivery_tag, options.fetch(:requeue, false))
+      def handleCancelOk(consumer_tag)
+        @cancelled = true
+      end
+
+      def start
       end
 
-      def delivery_tag
-        @envelope.delivery_tag
+      def deliver(headers, message)
+        raise NotImplementedError, 'To be implemented by a subclass'
       end
 
-      def routing_key
-        @envelope.routing_key
+      def cancel
+        channel.basic_cancel(consumer_tag)
+        @cancelling = true
       end
     end
 
-    module Subscriber
-      def start
-        # to be implemented by the host class
+    private
+
+    class CallbackConsumer < BaseConsumer
+      def initialize(channel, callback)
+        super(channel)
+        @callback = callback
+        @callback_arity = @callback.arity
+        @cancelled = false
+        @cancelling = false
       end
 
-      def on_message(&block)
-        raise ArgumentError, 'Message listener already registered for this subscriber' if @subscriber
-        @subscriber = block
-        start
+      def callback(headers, message)
+        if @callback_arity == 2
+          @callback.call(headers, message)
+        else
+          @callback.call(message)
+        end
+      end
+    end
+
+    class AsyncCallbackConsumer < CallbackConsumer
+      def initialize(channel, callback, executor)
+        super(channel, callback)
+        @executor = executor
+        @tasks = []
       end
 
-      def handle_message(consumer_tag, envelope, properties, body_bytes)
-        body = String.from_java_bytes(body_bytes)
-        case @subscriber.arity
-        when 2 then @subscriber.call(Headers.new(@channel, consumer_tag, envelope, properties), body)
-        when 1 then @subscriber.call(body)
-        else raise ArgumentError, 'Consumer callback wants no arguments'
+      def deliver(headers, message)
+        unless @executor.shutdown?
+          @executor.submit do
+            callback(headers, message)
+          end
         end
       end
     end
 
-    class BlockingSubscriber
-      include Subscriber
+    class BlockingCallbackConsumer < CallbackConsumer
+      include JavaConcurrent
+
+      def initialize(channel, buffer_size, callback)
+        super(channel, callback)
+        if buffer_size
+          @internal_queue = ArrayBlockingQueue.new(buffer_size)
+        else
+          @internal_queue = LinkedBlockingQueue.new
+        end
+      end
+
+      def start
+        interrupted = false
+        until @cancelled || JavaConcurrent::Thread.current_thread.interrupted?
+          begin
+            pair = @internal_queue.take
+            callback(*pair) if pair
+          rescue InterruptedException => e
+            interrupted = true
+          end
+        end
+        while (pair = @internal_queue.poll)
+          callback(*pair)
+        end
+        if interrupted
+          JavaConcurrent::Thread.current_thread.interrupt
+        end
+      end
+
+      def deliver(*pair)
+        if @cancelling || @cancelled || JavaConcurrent::Thread.current_thread.interrupted?
+          @internal_queue.offer(pair)
+        else
+          begin
+            @internal_queue.put(pair)
+          rescue InterruptedException => e
+            JavaConcurrent::Thread.current_thread.interrupt
+          end
+        end
+      end
+    end
 
-      attr_reader :consumer
+    class Headers
+      attr_reader :channel, :consumer_tag, :envelope, :properties
 
-      def initialize(channel, subscription)
+      def initialize(channel, consumer_tag, envelope, properties)
         @channel = channel
-        @subscription = subscription
-        @consumer = QueueingConsumer.new(@channel)
+        @consumer_tag = consumer_tag
+        @envelope = envelope
+        @properties = properties
       end
 
-      def consumer_tag
-        @consumer.consumer_tag
+      def ack(options={})
+        @channel.basic_ack(delivery_tag, options.fetch(:multiple, false))
       end
 
-      def start
-        super
-        while delivery = @consumer.next_delivery
-          result = handle_message(@consumer.consumer_tag, delivery.envelope, delivery.properties, delivery.body)
-          if result == :cancel
-            @subscription.cancel
-            while delivery = @consumer.next_delivery(0)
-              handle_message(@consumer.consumer_tag, delivery.envelope, delivery.properties, delivery.body)
-            end
-            break
-          end
+      def reject(options={})
+        @channel.basic_reject(delivery_tag, options.fetch(:requeue, false))
+      end
+
+      begin :envelope_delegation
+        [
+          :delivery_tag,
+          :routing_key,
+          :redeliver,
+          :exchange
+        ].each do |envelope_property|
+          define_method(envelope_property) { @envelope.__send__(envelope_property) }
+        end
+
+        alias_method :redelivered?, :redeliver
+      end
+
+      begin :message_properties_delegation
+        [
+          :content_encoding,
+          :content_type,
+          :content_encoding,
+          :headers,
+          :delivery_mode,
+          :priority,
+          :correlation_id,
+          :reply_to,
+          :expiration,
+          :message_id,
+          :timestamp,
+          :type,
+          :user_id,
+          :app_id,
+          :cluster_id
+        ].each do |properties_property|
+          define_method(properties_property) { @properties.__send__(properties_property) }
+        end
+
+        def persistent?
+          persistent == 2
         end
       end
     end

data/lib/hot_bunnies/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module HotBunnies
-  VERSION = '1.3.8'
+  VERSION = '1.4.0'
 end

data/lib/hot_bunnies.rb

@@ -6,13 +6,11 @@ require 'ext/rabbitmq-client'
 
 
 module HotBunnies
-  import com.rabbitmq.client.ConnectionFactory
-  import com.rabbitmq.client.Connection
-  import com.rabbitmq.client.Channel
-  import com.rabbitmq.client.DefaultConsumer
-  import com.rabbitmq.client.QueueingConsumer
-
-  import com.rabbitmq.client.AMQP
+  java_import 'com.rabbitmq.client.ConnectionFactory'
+  java_import 'com.rabbitmq.client.Connection'
+  java_import 'com.rabbitmq.client.Channel'
+  java_import 'com.rabbitmq.client.DefaultConsumer'
+  java_import 'com.rabbitmq.client.AMQP'
 
   def self.connect(options={})
     cf = ConnectionFactory.new
@@ -28,10 +26,21 @@ module HotBunnies
     cf.requested_heartbeat = heartbeat_from(options)          if include_heartbeat?(options)
     cf.connection_timeout  = connection_timeout_from(options) if include_connection_timeout?(options)
 
+    tls = (options[:ssl] || options[:tls])
+    case tls
+    when true then
+      cf.use_ssl_protocol
+    when String then
+      if options[:trust_manager]
+        cf.use_ssl_protocol(tls, options[:trust_manager])
+      else
+        cf.use_ssl_protocol(tls)
+      end
+    end
+
     cf.new_connection
   end
 
-
   protected
 
   def self.hostname_from(options)

data/spec/integration/basic_consume_spec.rb

@@ -1,5 +1,104 @@
 require "spec_helper"
 
+
+describe 'A consumer of a queue' do
+  let(:connection) { HotBunnies.connect }
+  let(:channel)    { connection.create_channel }
+
+  after :each do
+    channel.close
+    connection.close
+  end
+
+  it 'receives messages until cancelled' do
+    exchange = connection.create_channel.default_exchange
+    queue = connection.create_channel.queue("", :auto_delete => true)
+    subscription = queue.subscribe
+
+    messages = []
+    consumer_exited = false
+
+    consumer_thread = Thread.new do
+      subscription.each do |headers, message|
+        messages << message
+        sleep 0.1
+      end
+      consumer_exited = true
+    end
+
+    publisher_thread = Thread.new do
+      20.times do
+        exchange.publish('hello world', :routing_key => queue.name)
+        sleep 0.01
+      end
+    end
+
+    sleep 0.2
+
+    subscription.cancel
+
+    consumer_thread.join
+    publisher_thread.join
+
+    messages.should_not be_empty
+    consumer_exited.should be_true
+  end
+end
+
+describe "Multiple non-exclusive consumers per queue" do
+  let(:connection) { HotBunnies.connect }
+  let(:channel)    { connection.create_channel }
+
+  after :each do
+    channel.close
+    connection.close
+  end
+
+  context "on the same channel (so prefetch levels won't affect message distribution)" do
+    it "have messages distributed to them in the round robin manner" do
+      n                = 100
+      mailbox1         = []
+      mailbox2         = []
+      mailbox3         = []
+
+      all_received     = java.util.concurrent.CountDownLatch.new(n)
+      consumer_channel = connection.create_channel
+
+      queue            = channel.queue("", :auto_delete => true)
+
+      consumer1        = queue.subscribe(:blocking => false) do |metadata, payload|
+        mailbox1 << payload
+        all_received.count_down
+      end
+      consumer2        = queue.subscribe(:blocking => false) do |metadata, payload|
+        mailbox2 << payload
+        all_received.count_down
+      end
+      consumer3        = queue.subscribe(:blocking => false) do |metadata, payload|
+        mailbox3 << payload
+        all_received.count_down
+      end
+
+
+      sleep 2.0 # let consumers in other threads start.
+      n.times do |i|
+        channel.default_exchange.publish("Message #{i}", :routing_key => queue.name)
+      end
+
+      all_received.await
+
+      mailbox1.size.should >= 33
+      mailbox2.size.should >= 33
+      mailbox3.size.should >= 33
+
+      consumer1.shutdown!
+      consumer2.shutdown!
+      consumer3.shutdown!
+    end
+  end
+end
+
+
 describe "Queue consumer" do
   let(:connection) { HotBunnies.connect }
   let(:channel)    { connection.create_channel }

data/spec/integration/connection_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 
-describe "HotBunnies" do
+describe "HotBunnies.connect" do
 
   #
   # Examples
@@ -16,4 +16,11 @@ describe "HotBunnies" do
     c1 = HotBunnies.connect(:connection_timeout => 3)
     c1.close
   end
+
+  if !ENV["CI"] && ENV["TLS_TESTS"]
+    it "supports TLS w/o custom protocol or trust manager" do
+      c1 = HotBunnies.connect(:tls => true, :port => 5671)
+      c1.close
+    end    
+  end
 end

data/spec/integration/error_handling_by_consumers_spec.rb

@@ -0,0 +1,97 @@
+require "spec_helper"
+
+describe "An AMQP consumer that catches exceptions" do
+  let(:connection) { HotBunnies.connect }
+  let(:channel)    { connection.create_channel }
+
+  after :each do
+    channel.close
+    connection.close
+  end
+
+  it "stays up" do
+    mailbox  = []
+    exchange = channel.exchange("hot_bunnies.exchanges.fanout#{Time.now.to_i}", :type => :fanout, :auto_delete => true)
+    queue    = channel.queue("", :auto_delete => true)
+
+    queue.bind(exchange)
+    consumer = queue.subscribe(:blocking => false) do |meta, payload|
+      n = meta.properties.headers['X-Number']
+
+      begin
+        if n.odd?
+          raise "A failure"
+        else
+          mailbox << payload
+        end
+      rescue Exception => e
+        # no-op
+      end
+    end
+
+    25.times do |i|
+      exchange.publish("Message ##{i}", :routing_key => "xyz", :properties => {
+                         :headers => {
+                           'X-Number' => i
+                         }
+                       })
+    end
+
+    sleep(0.5)
+
+    mc, cc = queue.status
+    mc.should == 0
+
+    mailbox.size.should == 13
+    consumer.shutdown!
+  end
+end
+
+
+
+
+describe "An AMQP consumer that DOES NOT catch exceptions" do
+  let(:connection) { HotBunnies.connect }
+  let(:channel)    { connection.create_channel }
+
+  after :each do
+    channel.close
+    connection.close
+  end
+
+  it "becomes inactive when the channels prefetch is filled with unacked messages" do
+    mailbox  = []
+    exchange = channel.exchange("hot_bunnies.exchanges.fanout#{Time.now.to_i}#{rand}", :type => :fanout, :auto_delete => true)
+    queue    = channel.queue("", :auto_delete => true)
+
+    channel.prefetch = 5
+
+    queue.bind(exchange)
+    consumer = queue.subscribe(:blocking => false, :ack => true) do |meta, payload|
+      n = meta.properties.headers['X-Number']
+
+      if n.odd?
+        raise "A failure"
+      else
+        mailbox << payload
+        meta.ack
+      end
+    end
+
+    25.times do |i|
+      exchange.publish("Message ##{i}", :routing_key => "xyz", :properties => {
+                         :headers => {
+                           'X-Number' => i
+                         }
+                       })
+    end
+
+    sleep(0.5)
+
+    message_count, _ = queue.status
+    message_count.should == 15
+
+    mailbox.size.should == 5
+    consumer.shutdown!
+  end
+end

data/spec/integration/message_metadata_access_spec.rb

@@ -0,0 +1,94 @@
+require "spec_helper"
+
+describe "An AMQP consumer" do
+  let(:connection) { HotBunnies.connect }
+  let(:channel)    { connection.create_channel }
+
+  after :each do
+    channel.close
+    connection.close
+  end
+
+  it "can access message metadata (both message properties and delivery information)" do
+    latch    = java.util.concurrent.CountDownLatch.new(1)
+    queue    = channel.queue("", :auto_delete => true)
+    exchange = channel.exchange("amq.fanout", :type => :fanout)
+
+    queue.bind(exchange, :routing_key => "hotbunnies.key")
+
+    @now     = Time.now
+    @payload = "Hello, world!"
+    @meta    = nil
+
+    consumer = queue.subscribe(:blocking => false) do |metadata, payload|
+      begin
+        # we will run assertions on the main thread because RSpec uses exceptions
+        # for its purposes every once in a while. MK.
+        @meta = metadata
+      rescue Exception => e
+        e.print_stack_trace
+      ensure
+        latch.count_down
+      end
+    end
+
+    exchange.publish(@payload,
+                     :properties => { :app_id      => "hotbunnies.tests",
+                       :priority    => 8,
+                       :type        => "kinda.checkin",
+                       # headers table keys can be anything
+                       :headers     => {
+                         "coordinates" => {
+                           "latitude"  => 59.35,
+                           "longitude" => 18.066667
+                         },
+                         "time"         => @now,
+                         "participants" => 11,
+                         "venue"        => "Stockholm",
+                         "true_field"   => true,
+                         "false_field"  => false,
+                         "nil_field"    => nil,
+                         "ary_field"    => ["one", 2.0, 3, [{ "abc" => 123 }]]
+                       },
+                       :timestamp        => @now,
+                       :reply_to         => "a.sender",
+                       :correlation_id   => "r-1",
+                       :message_id       => "m-1",
+                       :content_type     => "application/octet-stream",
+                       # just an example. MK.
+                       :content_encoding => "zip/zap"
+                     },
+                     :routing_key    => "hotbunnies.key")
+    latch.await
+
+    @meta.routing_key.should  == "hotbunnies.key"
+    @meta.content_type.should == "application/octet-stream"
+    @meta.content_encoding.should == "zip/zap"
+    @meta.priority.should == 8
+
+    time = Time.at(@meta.headers["time"].getTime/1000)
+    time.to_i.should == @now.to_i
+
+    @meta.headers["coordinates"]["latitude"].should    == 59.35
+    @meta.headers["participants"].should == 11
+    @meta.headers["true_field"].should == true
+    @meta.headers["false_field"].should == false
+    @meta.headers["nil_field"].should be_nil
+
+    @meta.timestamp.should == Time.at(@now.to_i)
+    @meta.type.should == "kinda.checkin"
+    @meta.consumer_tag.should_not be_nil
+    @meta.consumer_tag.should_not be_empty
+    @meta.delivery_tag.should == 1
+    @meta.reply_to.should == "a.sender"
+    @meta.correlation_id.should == "r-1"
+    @meta.message_id.should == "m-1"
+    @meta.should_not be_redelivered
+
+    @meta.app_id.should == "hotbunnies.tests"
+    @meta.exchange.should == "amq.fanout"
+
+    consumer.shut_down!
+  end
+end
+

data/spec/spec_helper.rb

@@ -13,4 +13,6 @@ when "1.8.7" then
 end
 
 
-require "hot_bunnies"
+require "hot_bunnies"
+
+puts "Running on Ruby #{RUBY_VERSION}."

metadata

@@ -2,7 +2,7 @@
 name: hot_bunnies
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.3.8
+  version: 1.4.0
 platform: java
 authors: 
   - Theo Hultberg
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-07 00:00:00 Z
+date: 2012-11-14 00:00:00 Z
 dependencies: []
 
 description: A object oriented interface to RabbitMQ that uses the Java driver under the hood
@@ -27,6 +27,7 @@ files:
   - .gitignore
   - .rvmrc
   - .travis.yml
+  - ChangeLog.md
   - Gemfile
   - LICENSE
   - README.md
@@ -45,8 +46,10 @@ files:
   - spec/integration/alternate_exchanges_spec.rb
   - spec/integration/basic_consume_spec.rb
   - spec/integration/connection_spec.rb
+  - spec/integration/error_handling_by_consumers_spec.rb
   - spec/integration/exchange_bind_spec.rb
   - spec/integration/exchange_declare_spec.rb
+  - spec/integration/message_metadata_access_spec.rb
   - spec/integration/publisher_confirmations_spec.rb
   - spec/integration/queue_bind_spec.rb
   - spec/integration/queue_declare_spec.rb