hutch 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5f37cc28b7b467cec1505466b964a76af226d50
-  data.tar.gz: 7ced0eb70ab2c5a7e1765a4e58f7c9f24dae36d9
+  metadata.gz: ae290b72e28ca5ca72f8014fe5688e35b4c31716
+  data.tar.gz: 2a817eece4ab761d6558255395e123d5cea19ea0
 SHA512:
-  metadata.gz: ac2181c5109f698791668e589339a898aab136d62cb5eb05236fcf0c875776e05c00142e6bd6fb1279eb2adc2069df4d00246ec9d0a77052045d099dab8e7485
-  data.tar.gz: c7df71753321a9ed150e1eb56e6e9a994386ae5fd5e5ac81ec33c20b7da8075f587d56e6a2adb83663f939e15552f4991d4d5852e2a9713cbe5c08620812a63c
+  metadata.gz: b929abefa84452c0b5a74fc2dcfc5f2918fb752d80f97b77e97cf4d0fbfad99e119cd09f3251f17edf1450ee7ccc063a76efaa9e31d318526aabf5a147779b84
+  data.tar.gz: c5c04b649c3ea2333ff86273d7d17a1d843a02e782bd981d19ef2ea4025286761b4b96fd8200026ce2f503108d588c0aa61f204997632f02ac467ac5fa4633c3

data/.travis.yml

@@ -3,5 +3,9 @@ rvm:
   - "2.2"
   - "2.1"
   - "2.0"
+  - "jruby-9.0.0.0"
 services:
   - rabbitmq
+matrix:
+  allow_failures:
+    - rvm: jruby-9.0.0.0

data/CHANGELOG.md

@@ -1,6 +1,28 @@
 ## 0.18.0 — (unreleased)
 
-No changes yet.
+### JRuby Support (Using March Hare)
+
+Hutch will now use March Hare when running on JRuby.
+This will yield significant throughput and core utilisation
+improvements for workloads with many and/or busy consumers.
+
+Contributed by Teodor Pripoae.
+
+
+### Configurable Consumer Thread Pool Size
+
+`:consumer_pool_size` is a new option (defaults to `1`) which defines
+Bunny consumer work pool size.
+
+Contributed by Derek Kastner.
+
+### Bunny Logger Option
+
+`:client_logger` is a new option that allows
+for configuring loggerd used by Bunny, the underlying
+RabbitMQ client library.
+
+Contributed by Nate Salisbury.
 
 
 ## 0.17.0 — July 19th, 2015

data/README.md

@@ -116,6 +116,13 @@ logger rather than `stdout`, add this to `config/initializers/hutch.rb`
 Hutch::Logging.logger = Rails.logger
 ```
 
+A logger can be set for the client by adding this config before calling `Hutch.connect`
+
+```ruby
+client_logger = Logger.new("/path/to/bunny.log")
+Hutch::Config.set(:client_logger, client_logger)
+```
+
 See this [RabbitMQ tutorial on topic exchanges](http://www.rabbitmq.com/tutorials/tutorial-five-ruby.html)
 to learn more.
 

data/hutch.gemspec

@@ -1,7 +1,13 @@
 require File.expand_path('../lib/hutch/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.add_runtime_dependency 'bunny', '>= 1.7.0'
+  if defined?(JRUBY_VERSION)
+    gem.platform = 'java'
+    gem.add_runtime_dependency 'march_hare', '>= 2.11.0'
+  else
+    gem.platform = Gem::Platform::RUBY
+    gem.add_runtime_dependency 'bunny', '>= 1.7.0'
+  end
   gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
   gem.add_runtime_dependency 'multi_json', '~> 1.5'
   gem.add_runtime_dependency 'activesupport', '>= 3.0'

data/lib/hutch.rb

@@ -1,3 +1,4 @@
+require 'hutch/adapter'
 require 'hutch/consumer'
 require 'hutch/worker'
 require 'hutch/broker'
@@ -57,4 +58,3 @@ module Hutch
     broker.publish(*args)
   end
 end
-

data/lib/hutch/adapter.rb

@@ -0,0 +1,11 @@
+if defined?(JRUBY_VERSION)
+  require 'hutch/adapters/march_hare'
+  module Hutch
+    Adapter = Adapters::MarchHareAdapter
+  end
+else
+  require 'hutch/adapters/bunny'
+  module Hutch
+    Adapter = Adapters::BunnyAdapter
+  end
+end

data/lib/hutch/adapters/bunny.rb

@@ -0,0 +1,33 @@
+require 'bunny'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class BunnyAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = Bunny::Session::DEFAULT_VHOST
+
+      ConnectionRefused = Bunny::TCPConnectionFailed
+      PreconditionFailed = Bunny::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :create_channel, :open?
+
+      def initialize(opts={})
+        @connection = Bunny.new(opts)
+      end
+
+      def self.decode_message(delivery_info, properties, payload)
+        [delivery_info, properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch(prefetch) if prefetch
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/hutch/adapters/march_hare.rb

@@ -0,0 +1,37 @@
+require 'march_hare'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class MarchHareAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = "/"
+
+      ConnectionRefused = MarchHare::ConnectionRefused
+      PreconditionFailed = MarchHare::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :open?
+
+      def initialize(opts = {})
+        @connection = MarchHare.connect(opts)
+      end
+
+      def self.decode_message(delivery_info, payload)
+        [delivery_info, delivery_info.properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch = prefetch if prefetch
+      end
+
+      def create_channel(n = nil, consumer_pool_size = 1)
+        @connection.create_channel(n)
+      end
+
+      def current_timestamp
+        Time.now
+      end
+    end
+  end
+end

data/lib/hutch/broker.rb

@@ -1,4 +1,3 @@
-require 'bunny'
 require 'carrot-top'
 require 'securerandom'
 require 'hutch/logging'
@@ -61,58 +60,22 @@ module Hutch
     end
 
     def open_connection!
-      if @config[:uri] && !@config[:uri].empty?
-        u = URI.parse(@config[:uri])
-
-        @config[:mq_host]     = u.host
-        @config[:mq_port]     = u.port
-        @config[:mq_vhost]    = u.path.sub(/^\//, "")
-        @config[:mq_username] = u.user
-        @config[:mq_password] = u.password
-        @config[:mq_tls]      = u.scheme == "amqps"
-      end
-
-      tls                = @config[:mq_tls]
-      host               = @config[:mq_host]
-      port               = @config.to_hash.fetch(:mq_port, (tls ? 5671 : 5672))
-      vhost              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
-                             @config[:mq_vhost]
-                           else
-                             Bunny::Session::DEFAULT_VHOST
-                           end
-      username           = @config[:mq_username]
-      password           = @config[:mq_password]
-      tls_key            = @config[:mq_tls_key]
-      tls_cert           = @config[:mq_tls_cert]
-      heartbeat          = @config[:heartbeat]
-      connection_timeout = @config[:connection_timeout]
-      read_timeout       = @config[:read_timeout]
-      write_timeout      = @config[:write_timeout]
-
-      scheme             = tls ? "amqps" : "amqp"
-      sanitized_uri      = "#{scheme}://#{username}@#{host}:#{port}/#{vhost.sub(/^\//, '')}"
       logger.info "connecting to rabbitmq (#{sanitized_uri})"
-      @connection = Bunny.new(host: host, port: port, vhost: vhost,
-                              tls: tls, tls_key: tls_key, tls_cert: tls_cert,
-                              username: username, password: password,
-                              heartbeat: heartbeat, automatically_recover: true,
-                              network_recovery_interval: 1,
-                              connection_timeout: connection_timeout,
-                              read_timeout: read_timeout,
-                              write_timeout: write_timeout)
+
+      @connection = Hutch::Adapter.new(connection_params)
 
       with_bunny_connection_handler(sanitized_uri) do
         @connection.start
       end
 
-      logger.info "connected to RabbitMQ at #{host} as #{username}"
+      logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
       @connection
     end
 
     def open_channel!
-      logger.info 'opening rabbitmq channel'
-      @channel = connection.create_channel.tap do |ch|
-        ch.prefetch(@config[:channel_prefetch]) if @config[:channel_prefetch]
+      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
+      @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
+        @connection.prefetch_channel(ch, @config[:channel_prefetch])
         if @config[:publisher_confirms] || @config[:force_publisher_confirms]
           logger.info 'enabling publisher confirms'
           ch.confirm_select
@@ -204,12 +167,16 @@ module Hutch
     end
 
     def stop
-      # Enqueue a failing job that kills the consumer loop
-      channel_work_pool.shutdown
-      # Give `timeout` seconds to jobs that are still being processed
-      channel_work_pool.join(@config[:graceful_exit_timeout])
-      # If after `timeout` they are still running, they are killed
-      channel_work_pool.kill
+      if defined?(JRUBY_VERSION)
+        channel.close
+      else
+        # Enqueue a failing job that kills the consumer loop
+        channel_work_pool.shutdown
+        # Give `timeout` seconds to jobs that are still being processed
+        channel_work_pool.join(@config[:graceful_exit_timeout])
+        # If after `timeout` they are still running, they are killed
+        channel_work_pool.kill
+      end
     end
 
     def requeue(delivery_tag)
@@ -233,7 +200,7 @@ module Hutch
 
       non_overridable_properties = {
         routing_key: routing_key,
-        timestamp: Time.now.to_i,
+        timestamp: @connection.current_timestamp,
         content_type: 'application/json'
       }
       properties[:message_id] ||= generate_id
@@ -286,6 +253,54 @@ module Hutch
       end
     end
 
+    def connection_params
+      parse_uri
+
+      {}.tap do |params|
+        params[:host]               = @config[:mq_host]
+        params[:port]               = @config[:mq_port]
+        params[:vhost]              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
+                                        @config[:mq_vhost]
+                                      else
+                                        Hutch::Adapter::DEFAULT_VHOST
+                                      end
+        params[:username]           = @config[:mq_username]
+        params[:password]           = @config[:mq_password]
+        params[:tls]                = @config[:mq_tls]
+        params[:tls_key]            = @config[:mq_tls_key]
+        params[:tls_cert]           = @config[:mq_tls_cert]
+        params[:heartbeat]          = @config[:heartbeat]
+        params[:connection_timeout] = @config[:connection_timeout]
+        params[:read_timeout]       = @config[:read_timeout]
+        params[:write_timeout]      = @config[:write_timeout]
+
+
+        params[:automatically_recover] = true
+        params[:network_recovery_interval] = 1
+
+        params[:client_logger] = @config[:client_logger] if @config[:client_logger]
+      end
+    end
+
+    def parse_uri
+      return unless @config[:uri] && !@config[:uri].empty?
+
+      u = URI.parse(@config[:uri])
+
+      @config[:mq_host]     = u.host
+      @config[:mq_port]     = u.port
+      @config[:mq_vhost]    = u.path.sub(/^\//, "")
+      @config[:mq_username] = u.user
+      @config[:mq_password] = u.password
+    end
+
+    def sanitized_uri
+      p = connection_params
+      scheme = p[:tls] ? "amqps" : "amqp"
+
+      "#{scheme}://#{p[:username]}@#{p[:host]}:#{p[:port]}/#{p[:vhost].sub(/^\//, '')}"
+    end
+
     def with_authentication_error_handler
       yield
     rescue Net::HTTPServerException => ex
@@ -306,7 +321,7 @@ module Hutch
 
     def with_bunny_precondition_handler(item)
       yield
-    rescue Bunny::PreconditionFailed => ex
+    rescue Hutch::Adapter::PreconditionFailed => ex
       logger.error ex.message
       s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
           "Perhaps it is being redeclared with non-matching attributes"
@@ -315,7 +330,7 @@ module Hutch
 
     def with_bunny_connection_handler(uri)
       yield
-    rescue Bunny::TCPConnectionFailed => ex
+    rescue Hutch::Adapter::ConnectionRefused => ex
       logger.error "amqp connection error: #{ex.message.downcase}"
       raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
     end
@@ -328,6 +343,10 @@ module Hutch
       @channel.work_pool
     end
 
+    def consumer_pool_size
+      @config[:consumer_pool_size]
+    end
+
     def generate_id
       SecureRandom.uuid
     end
@@ -337,4 +356,3 @@ module Hutch
     end
   end
 end
-

data/lib/hutch/config.rb

@@ -50,6 +50,9 @@ module Hutch
         # to finish its job when gracefully exiting Hutch, before
         # it's killed.
         graceful_exit_timeout: 11,
+        client_logger: nil,
+
+        consumer_pool_size: 1,
       }.merge(params)
     end
 

data/lib/hutch/version.rb

@@ -1,4 +1,4 @@
 module Hutch
-  VERSION = '0.17.0'.freeze
+  VERSION = '0.18.0'.freeze
 end
 

data/lib/hutch/worker.rb

@@ -21,9 +21,20 @@ module Hutch
       # Set up signal handlers for graceful shutdown
       register_signal_handlers
 
-      # Take a break from Thread#join every 0.1 seconds to check if we've
-      # been sent any signals
-      handle_signals until @broker.wait_on_threads(0.1)
+      main_loop
+    end
+
+    def main_loop
+      if defined?(JRUBY_VERSION)
+        # Binds shutdown listener to notify main thread if channel was closed
+        bind_shutdown_handler
+
+        handle_signals until shutdown_not_called?(0.1)
+      else
+        # Take a break from Thread#join every 0.1 seconds to check if we've
+        # been sent any signals
+        handle_signals until @broker.wait_on_threads(0.1)
+      end
     end
 
     # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
@@ -53,6 +64,23 @@ module Hutch
       @broker.stop
     end
 
+    # Binds shutdown handler, called if channel is closed or network Failed
+    def bind_shutdown_handler
+      @broker.channel.on_shutdown do
+        Thread.main[:shutdown_received] = true
+      end
+    end
+
+    # Checks if shutdown handler was called, then sleeps for interval
+    def shutdown_not_called?(interval)
+      if Thread.main[:shutdown_received]
+        true
+      else
+        sleep(interval)
+        false
+      end
+    end
+
     # Set up the queues for each of the worker's consumers.
     def setup_queues
       logger.info 'setting up queues'
@@ -65,7 +93,8 @@ module Hutch
       queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
       @broker.bind_queue(queue, consumer.routing_keys)
 
-      queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
+      queue.subscribe(manual_ack: true) do |*args|
+        delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
         handle_message(consumer, delivery_info, properties, payload)
       end
     end

data/spec/hutch/broker_spec.rb

@@ -58,20 +58,35 @@ describe Hutch::Broker do
       before { broker.set_up_amqp_connection }
       after  { broker.disconnect }
 
-      describe '#connection' do
+      describe '#connection', adapter: :bunny do
         subject { super().connection }
-        it { is_expected.to be_a Bunny::Session }
+        it { is_expected.to be_a Hutch::Adapters::BunnyAdapter }
       end
 
-      describe '#channel' do
+      describe '#connection', adapter: :march_hare do
+        subject { super().connection }
+        it { is_expected.to be_a Hutch::Adapters::MarchHareAdapter }
+      end
+
+      describe '#channel', adapter: :bunny do
         subject { super().channel }
         it { is_expected.to be_a Bunny::Channel }
       end
 
-      describe '#exchange' do
+      describe '#channel', adapter: :march_hare do
+        subject { super().channel }
+        it { is_expected.to be_a MarchHare::Channel }
+      end
+
+      describe '#exchange', adapter: :bunny do
         subject { super().exchange }
         it { is_expected.to be_a Bunny::Exchange }
       end
+
+      describe '#exchange', adapter: :march_hare do
+        subject { super().exchange }
+        it { is_expected.to be_a MarchHare::Exchange }
+      end
     end
 
     context 'when given invalid details' do
@@ -86,11 +101,17 @@ describe Hutch::Broker do
       before { config[:channel_prefetch] = prefetch_value }
       after  { broker.disconnect }
 
-      it "set's channel's prefetch" do
+      it "set's channel's prefetch", adapter: :bunny do
         expect_any_instance_of(Bunny::Channel).
           to receive(:prefetch).with(prefetch_value)
         broker.set_up_amqp_connection
       end
+
+      it "set's channel's prefetch", adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:prefetch=).with(prefetch_value)
+        broker.set_up_amqp_connection
+      end
     end
 
     context 'with force_publisher_confirms set' do
@@ -98,11 +119,17 @@ describe Hutch::Broker do
       before { config[:force_publisher_confirms] = force_publisher_confirms_value }
       after  { broker.disconnect }
 
-      it 'waits for confirmation' do
+      it 'waits for confirmation', adapter: :bunny do
         expect_any_instance_of(Bunny::Channel).
           to receive(:confirm_select)
         broker.set_up_amqp_connection
       end
+
+      it 'waits for confirmation', adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:confirm_select)
+        broker.set_up_amqp_connection
+      end
     end
   end
 
@@ -213,7 +240,7 @@ describe Hutch::Broker do
     end
   end
 
-  describe '#stop' do
+  describe '#stop', adapter: :bunny do
     let(:thread_1) { double('Thread') }
     let(:thread_2) { double('Thread') }
     let(:work_pool) { double('Bunny::ConsumerWorkPool') }
@@ -232,6 +259,20 @@ describe Hutch::Broker do
     end
   end
 
+  describe '#stop', adapter: :march_hare do
+    let(:channel) { double('MarchHare::Channel')}
+
+    before do
+      allow(broker).to receive(:channel).and_return(channel)
+    end
+
+    it 'gracefully stops the channel' do
+      expect(channel).to receive(:close)
+
+      broker.stop
+    end
+  end
+
   describe '#publish' do
     context 'with a valid connection' do
       before { broker.set_up_amqp_connection }
@@ -287,11 +328,17 @@ describe Hutch::Broker do
       end
 
       context 'with force_publisher_confirms not set in the config' do
-        it 'does not wait for confirms on the channel' do
+        it 'does not wait for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to_not receive(:wait_for_confirms)
           broker.publish('test.key', 'message')
         end
+
+        it 'does not wait for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to_not receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
       end
 
       context 'with force_publisher_confirms set in the config' do
@@ -301,11 +348,17 @@ describe Hutch::Broker do
           config[:force_publisher_confirms] = force_publisher_confirms_value
         end
 
-        it 'waits for confirms on the channel' do
+        it 'waits for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to receive(:wait_for_confirms)
           broker.publish('test.key', 'message')
         end
+
+        it 'waits for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
       end
     end
 
@@ -322,4 +375,3 @@ describe Hutch::Broker do
     end
   end
 end
-

data/spec/spec_helper.rb

@@ -16,6 +16,12 @@ require 'logger'
 RSpec.configure do |config|
   config.before(:all) { Hutch::Config.log_level = Logger::FATAL }
   config.raise_errors_for_deprecations!
+
+  if defined?(JRUBY_VERSION)
+    config.filter_run_excluding adapter: :bunny
+  else
+    config.filter_run_excluding adapter: :march_hare
+  end
 end
 
 # Constants (classes, etc) defined within a block passed to this method
@@ -32,4 +38,3 @@ end
 def deep_copy(obj)
   Marshal.load(Marshal.dump(obj))
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hutch
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Harry Marr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-19 00:00:00.000000000 Z
+date: 2015-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -117,6 +117,9 @@ files:
 - examples/producer.rb
 - hutch.gemspec
 - lib/hutch.rb
+- lib/hutch/adapter.rb
+- lib/hutch/adapters/bunny.rb
+- lib/hutch/adapters/march_hare.rb
 - lib/hutch/broker.rb
 - lib/hutch/cli.rb
 - lib/hutch/config.rb
@@ -167,7 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Easy inter-service communication using RabbitMQ.
@@ -185,3 +188,4 @@ test_files:
 - spec/hutch/worker_spec.rb
 - spec/hutch_spec.rb
 - spec/spec_helper.rb
+has_rdoc: