dkastner-hutch 0.17.1 → 0.19.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19db67e67c4be705a1941803cb05ac03357e32cf
-  data.tar.gz: 7e85becb93bb0c994b16f533a3cc7b7a34c4d0d6
+  metadata.gz: ca9c923df5ace74031777bc88f495b997123dbdb
+  data.tar.gz: a0e464a895e6c8f58dc58909ce3afb42e9a0e565
 SHA512:
-  metadata.gz: 6ccfebf8a648f1d33e7fc49bc7a03a0b7cac70a93cf3321ec1f1a5cb58018fd671f61ba76eef534aad60143b66eaf7edd1c2f726b3b217d5e2d5be680daca589
-  data.tar.gz: eca9fba818c940b01ca8969c8ad9794bc31935a22c980f6cedea833507aeeb8bf431b18e1079737cbfbe1f3ddf687b0b28aa0d54ef75095850d707ce52c92431
+  metadata.gz: fbc9df84841ec14bc014d255f7ba7ac097ff662ee93e345e9866582393ced2ccd0c5bf054b1358e5d808439d170f81b66e295c37dfe92ce528e3ef23eb908e09
+  data.tar.gz: 0fbf1b852344c3e42849725a1ceba069665efaafe6dcc20851e6fe18105e94cff5ff862af9188380c1da34c32adfecb118900b7db69caa9b43613f215095dbf1

data/.travis.yml

@@ -3,5 +3,9 @@ rvm:
   - "2.2"
   - "2.1"
   - "2.0"
+  - "jruby-9.0.0.0"
 services:
   - rabbitmq
+matrix:
+  allow_failures:
+    - rvm: jruby-9.0.0.0

data/CHANGELOG.md

@@ -1,4 +1,36 @@
-## 0.18.0 — (unreleased)
+## 0.19.0 — (unreleased)
+
+### multi_json Update
+
+Hutch now depends on multi_json `1.11.x`.
+
+### Bunny Update
+
+Bunny is updated to [2.1.0](http://blog.rubyrabbitmq.info/blog/2015/08/16/bunny-2-dot-1-0-is-released/).
+
+### More Bunny SSL Options
+
+`:mq_tls_ca_certificates` and `:mq_verify_peer` options will now be passed on to Bunny as `:tls_ca_certificates` and `:verify_peer` respectively.
+
+Contributed by Kennon Ballou.
+
+## 0.18.0 — August 16th, 2015
+
+### JRuby Support (Using March Hare)
+
+Hutch will now use March Hare when running on JRuby.
+This will yield significant throughput and core utilisation
+improvements for workloads with many and/or busy consumers.
+
+Contributed by Teodor Pripoae.
+
+
+### Configurable Consumer Thread Pool Size
+
+`:consumer_pool_size` is a new option (defaults to `1`) which defines
+Bunny consumer work pool size.
+
+Contributed by Derek Kastner.
 
 ### Bunny Logger Option
 

data/README.md

@@ -289,6 +289,8 @@ Known configuration parameters are:
  * `mq_tls`: should TLS be used? (default: `false`)
  * `mq_tls_cert`: path to client TLS certificate (public key)
  * `mq_tls_key`: path to client TLS private key
+ * `mq_tls_ca_certificates`: array of paths to CA keys (if not specified to Hutch, will default to Bunny defaults which are system-dependent)
+ * `mq_verify_peer`: should SSL certificate be verified? (default: `true`)
  * `require_paths`: array of paths to require
  * `autoload_rails`: should Hutch command line runner try to automatically load Rails environment files?
  * `daemonise`: should Hutch runner process daemonise?

data/hutch.gemspec

@@ -1,9 +1,15 @@
 require File.expand_path('../lib/hutch/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.add_runtime_dependency 'bunny', '>= 1.7.0'
+  if defined?(JRUBY_VERSION)
+    gem.platform = 'java'
+    gem.add_runtime_dependency 'march_hare', '>= 2.11.0'
+  else
+    gem.platform = Gem::Platform::RUBY
+    gem.add_runtime_dependency 'bunny', '>= 2.1.0'
+  end
   gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
-  gem.add_runtime_dependency 'multi_json', '~> 1.5'
+  gem.add_runtime_dependency 'multi_json', '~> 1.11.2'
   gem.add_runtime_dependency 'activesupport', '>= 3.0'
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'simplecov', '~> 0.7.1'

data/lib/hutch/acknowledgements/nack_on_all_failures.rb

@@ -0,0 +1,18 @@
+require 'hutch/logging'
+
+module Hutch
+  module Acknowledgements
+    class NackOnAllFailures
+      include Logging
+
+      def handle(delivery_info, properties, broker, ex)
+        prefix = "message(#{properties.message_id || '-'}): "
+        logger.debug "#{prefix} nacking message"
+
+        broker.nack delivery_info.delivery_tag
+
+        true
+      end
+    end
+  end
+end

data/lib/hutch/adapter.rb

@@ -0,0 +1,11 @@
+if defined?(JRUBY_VERSION)
+  require 'hutch/adapters/march_hare'
+  module Hutch
+    Adapter = Adapters::MarchHareAdapter
+  end
+else
+  require 'hutch/adapters/bunny'
+  module Hutch
+    Adapter = Adapters::BunnyAdapter
+  end
+end

data/lib/hutch/adapters/bunny.rb

@@ -0,0 +1,33 @@
+require 'bunny'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class BunnyAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = Bunny::Session::DEFAULT_VHOST
+
+      ConnectionRefused = Bunny::TCPConnectionFailed
+      PreconditionFailed = Bunny::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :create_channel, :open?
+
+      def initialize(opts={})
+        @connection = Bunny.new(opts)
+      end
+
+      def self.decode_message(delivery_info, properties, payload)
+        [delivery_info, properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch(prefetch) if prefetch
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/hutch/adapters/march_hare.rb

@@ -0,0 +1,37 @@
+require 'march_hare'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class MarchHareAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = "/"
+
+      ConnectionRefused = MarchHare::ConnectionRefused
+      PreconditionFailed = MarchHare::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :open?
+
+      def initialize(opts = {})
+        @connection = MarchHare.connect(opts)
+      end
+
+      def self.decode_message(delivery_info, payload)
+        [delivery_info, delivery_info.properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch = prefetch if prefetch
+      end
+
+      def create_channel(n = nil, consumer_pool_size = 1)
+        @connection.create_channel(n)
+      end
+
+      def current_timestamp
+        Time.now
+      end
+    end
+  end
+end

data/lib/hutch/broker.rb

@@ -1,4 +1,3 @@
-require 'bunny'
 require 'carrot-top'
 require 'securerandom'
 require 'hutch/logging'
@@ -63,7 +62,7 @@ module Hutch
     def open_connection!
       logger.info "connecting to rabbitmq (#{sanitized_uri})"
 
-      @connection = Bunny.new(connection_params)
+      @connection = Hutch::Adapter.new(connection_params)
 
       with_bunny_connection_handler(sanitized_uri) do
         @connection.start
@@ -75,8 +74,8 @@ module Hutch
 
     def open_channel!
       logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
-      @channel = connection.create_channel(nil, consumer_pool_size).tap do |ch|
-        ch.prefetch(@config[:channel_prefetch]) if @config[:channel_prefetch]
+      @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
+        @connection.prefetch_channel(ch, @config[:channel_prefetch])
         if @config[:publisher_confirms] || @config[:force_publisher_confirms]
           logger.info 'enabling publisher confirms'
           ch.confirm_select
@@ -168,12 +167,16 @@ module Hutch
     end
 
     def stop
-      # Enqueue a failing job that kills the consumer loop
-      channel_work_pool.shutdown
-      # Give `timeout` seconds to jobs that are still being processed
-      channel_work_pool.join(@config[:graceful_exit_timeout])
-      # If after `timeout` they are still running, they are killed
-      channel_work_pool.kill
+      if defined?(JRUBY_VERSION)
+        channel.close
+      else
+        # Enqueue a failing job that kills the consumer loop
+        channel_work_pool.shutdown
+        # Give `timeout` seconds to jobs that are still being processed
+        channel_work_pool.join(@config[:graceful_exit_timeout])
+        # If after `timeout` they are still running, they are killed
+        channel_work_pool.kill
+      end
     end
 
     def requeue(delivery_tag)
@@ -197,7 +200,7 @@ module Hutch
 
       non_overridable_properties = {
         routing_key: routing_key,
-        timestamp: Time.now.to_i,
+        timestamp: @connection.current_timestamp,
         content_type: 'application/json'
       }
       properties[:message_id] ||= generate_id
@@ -259,13 +262,17 @@ module Hutch
         params[:vhost]              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
                                         @config[:mq_vhost]
                                       else
-                                        Bunny::Session::DEFAULT_VHOST
+                                        Hutch::Adapter::DEFAULT_VHOST
                                       end
         params[:username]           = @config[:mq_username]
         params[:password]           = @config[:mq_password]
         params[:tls]                = @config[:mq_tls]
         params[:tls_key]            = @config[:mq_tls_key]
         params[:tls_cert]           = @config[:mq_tls_cert]
+        params[:verify_peer]        = @config[:mq_verify_peer]
+        if @config[:mq_tls_ca_certificates]
+          params[:tls_ca_certificates] = @config[:mq_tls_ca_certificates]
+        end
         params[:heartbeat]          = @config[:heartbeat]
         params[:connection_timeout] = @config[:connection_timeout]
         params[:read_timeout]       = @config[:read_timeout]
@@ -318,7 +325,7 @@ module Hutch
 
     def with_bunny_precondition_handler(item)
       yield
-    rescue Bunny::PreconditionFailed => ex
+    rescue Hutch::Adapter::PreconditionFailed => ex
       logger.error ex.message
       s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
           "Perhaps it is being redeclared with non-matching attributes"
@@ -327,7 +334,7 @@ module Hutch
 
     def with_bunny_connection_handler(uri)
       yield
-    rescue Bunny::TCPConnectionFailed => ex
+    rescue Hutch::Adapter::ConnectionRefused => ex
       logger.error "amqp connection error: #{ex.message.downcase}"
       raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
     end
@@ -353,4 +360,3 @@ module Hutch
     end
   end
 end
-

data/lib/hutch/config.rb

@@ -17,6 +17,8 @@ module Hutch
         mq_tls: false,
         mq_tls_cert: nil,
         mq_tls_key: nil,
+        mq_tls_ca_certificates: nil,
+        mq_verify_peer: true,
         mq_username: 'guest',
         mq_password: 'guest',
         mq_api_host: 'localhost',
@@ -30,6 +32,7 @@ module Hutch
         require_paths: [],
         autoload_rails: true,
         error_handlers: [Hutch::ErrorHandlers::Logger.new],
+        error_acknowledgements: [],
         tracer: Hutch::Tracers::NullTracer,
         namespace: nil,
         daemonise: false,

data/lib/hutch/version.rb

@@ -1,4 +1,4 @@
 module Hutch
-  VERSION = '0.17.1'.freeze
+  VERSION = '0.19.0.pre1'.freeze
 end
 

data/lib/hutch/worker.rb

@@ -1,6 +1,7 @@
 require 'hutch/message'
 require 'hutch/logging'
 require 'hutch/broker'
+require 'hutch/acknowledgements/nack_on_all_failures'
 require 'carrot-top'
 
 module Hutch
@@ -21,9 +22,20 @@ module Hutch
       # Set up signal handlers for graceful shutdown
       register_signal_handlers
 
-      # Take a break from Thread#join every 0.1 seconds to check if we've
-      # been sent any signals
-      handle_signals until @broker.wait_on_threads(0.1)
+      main_loop
+    end
+
+    def main_loop
+      if defined?(JRUBY_VERSION)
+        # Binds shutdown listener to notify main thread if channel was closed
+        bind_shutdown_handler
+
+        handle_signals until shutdown_not_called?(0.1)
+      else
+        # Take a break from Thread#join every 0.1 seconds to check if we've
+        # been sent any signals
+        handle_signals until @broker.wait_on_threads(0.1)
+      end
     end
 
     # Register handlers for SIG{QUIT,TERM,INT} to shut down the worker
@@ -53,6 +65,23 @@ module Hutch
       @broker.stop
     end
 
+    # Binds shutdown handler, called if channel is closed or network Failed
+    def bind_shutdown_handler
+      @broker.channel.on_shutdown do
+        Thread.main[:shutdown_received] = true
+      end
+    end
+
+    # Checks if shutdown handler was called, then sleeps for interval
+    def shutdown_not_called?(interval)
+      if Thread.main[:shutdown_received]
+        true
+      else
+        sleep(interval)
+        false
+      end
+    end
+
     # Set up the queues for each of the worker's consumers.
     def setup_queues
       logger.info 'setting up queues'
@@ -65,7 +94,8 @@ module Hutch
       queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
       @broker.bind_queue(queue, consumer.routing_keys)
 
-      queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
+      queue.subscribe(manual_ack: true) do |*args|
+        delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
         handle_message(consumer, delivery_info, properties, payload)
       end
     end
@@ -85,7 +115,7 @@ module Hutch
         with_tracing(consumer_instance).handle(message)
         broker.ack(delivery_info.delivery_tag)
       rescue StandardError => ex
-        broker.nack(delivery_info.delivery_tag)
+        acknowledge_error(delivery_info, properties, broker, ex)
         handle_error(properties.message_id, payload, consumer, ex)
       end
     end
@@ -100,11 +130,23 @@ module Hutch
       end
     end
 
+    def acknowledge_error(delivery_info, properties, broker, ex)
+      acks = error_acknowledgements +
+        [Hutch::Acknowledgements::NackOnAllFailures.new]
+      acks.find do |backend|
+        backend.handle(delivery_info, properties, broker, ex)
+      end
+    end
+
     def consumers=(val)
       if val.empty?
         logger.warn "no consumer loaded, ensure there's no configuration issue"
       end
       @consumers = val
     end
+
+    def error_acknowledgements
+      Hutch::Config[:error_acknowledgements]
+    end
   end
 end

data/lib/hutch.rb

@@ -1,3 +1,4 @@
+require 'hutch/adapter'
 require 'hutch/consumer'
 require 'hutch/worker'
 require 'hutch/broker'
@@ -57,4 +58,3 @@ module Hutch
     broker.publish(*args)
   end
 end
-

data/spec/hutch/broker_spec.rb

@@ -58,20 +58,35 @@ describe Hutch::Broker do
       before { broker.set_up_amqp_connection }
       after  { broker.disconnect }
 
-      describe '#connection' do
+      describe '#connection', adapter: :bunny do
         subject { super().connection }
-        it { is_expected.to be_a Bunny::Session }
+        it { is_expected.to be_a Hutch::Adapters::BunnyAdapter }
       end
 
-      describe '#channel' do
+      describe '#connection', adapter: :march_hare do
+        subject { super().connection }
+        it { is_expected.to be_a Hutch::Adapters::MarchHareAdapter }
+      end
+
+      describe '#channel', adapter: :bunny do
         subject { super().channel }
         it { is_expected.to be_a Bunny::Channel }
       end
 
-      describe '#exchange' do
+      describe '#channel', adapter: :march_hare do
+        subject { super().channel }
+        it { is_expected.to be_a MarchHare::Channel }
+      end
+
+      describe '#exchange', adapter: :bunny do
         subject { super().exchange }
         it { is_expected.to be_a Bunny::Exchange }
       end
+
+      describe '#exchange', adapter: :march_hare do
+        subject { super().exchange }
+        it { is_expected.to be_a MarchHare::Exchange }
+      end
     end
 
     context 'when given invalid details' do
@@ -86,11 +101,17 @@ describe Hutch::Broker do
       before { config[:channel_prefetch] = prefetch_value }
       after  { broker.disconnect }
 
-      it "set's channel's prefetch" do
+      it "set's channel's prefetch", adapter: :bunny do
         expect_any_instance_of(Bunny::Channel).
           to receive(:prefetch).with(prefetch_value)
         broker.set_up_amqp_connection
       end
+
+      it "set's channel's prefetch", adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:prefetch=).with(prefetch_value)
+        broker.set_up_amqp_connection
+      end
     end
 
     context 'with force_publisher_confirms set' do
@@ -98,11 +119,17 @@ describe Hutch::Broker do
       before { config[:force_publisher_confirms] = force_publisher_confirms_value }
       after  { broker.disconnect }
 
-      it 'waits for confirmation' do
+      it 'waits for confirmation', adapter: :bunny do
         expect_any_instance_of(Bunny::Channel).
           to receive(:confirm_select)
         broker.set_up_amqp_connection
       end
+
+      it 'waits for confirmation', adapter: :march_hare do
+        expect_any_instance_of(MarchHare::Channel).
+          to receive(:confirm_select)
+        broker.set_up_amqp_connection
+      end
     end
   end
 
@@ -213,7 +240,7 @@ describe Hutch::Broker do
     end
   end
 
-  describe '#stop' do
+  describe '#stop', adapter: :bunny do
     let(:thread_1) { double('Thread') }
     let(:thread_2) { double('Thread') }
     let(:work_pool) { double('Bunny::ConsumerWorkPool') }
@@ -232,6 +259,20 @@ describe Hutch::Broker do
     end
   end
 
+  describe '#stop', adapter: :march_hare do
+    let(:channel) { double('MarchHare::Channel')}
+
+    before do
+      allow(broker).to receive(:channel).and_return(channel)
+    end
+
+    it 'gracefully stops the channel' do
+      expect(channel).to receive(:close)
+
+      broker.stop
+    end
+  end
+
   describe '#publish' do
     context 'with a valid connection' do
       before { broker.set_up_amqp_connection }
@@ -287,11 +328,17 @@ describe Hutch::Broker do
       end
 
       context 'with force_publisher_confirms not set in the config' do
-        it 'does not wait for confirms on the channel' do
+        it 'does not wait for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to_not receive(:wait_for_confirms)
           broker.publish('test.key', 'message')
         end
+
+        it 'does not wait for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to_not receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
       end
 
       context 'with force_publisher_confirms set in the config' do
@@ -301,11 +348,17 @@ describe Hutch::Broker do
           config[:force_publisher_confirms] = force_publisher_confirms_value
         end
 
-        it 'waits for confirms on the channel' do
+        it 'waits for confirms on the channel', adapter: :bunny do
           expect_any_instance_of(Bunny::Channel).
             to receive(:wait_for_confirms)
           broker.publish('test.key', 'message')
         end
+
+        it 'waits for confirms on the channel', adapter: :march_hare do
+          expect_any_instance_of(MarchHare::Channel).
+            to receive(:wait_for_confirms)
+          broker.publish('test.key', 'message')
+        end
       end
     end
 
@@ -322,4 +375,3 @@ describe Hutch::Broker do
     end
   end
 end
-

data/spec/hutch/worker_spec.rb

@@ -62,6 +62,25 @@ describe Hutch::Worker do
       worker.handle_message(consumer, delivery_info, properties, payload)
     end
 
+    context 'when the consumer fails and a requeue is configured' do
+
+      it 'requeues the message' do
+        allow(consumer_instance).to receive(:process).and_raise('failed')
+        requeuer = double
+        allow(requeuer).to receive(:handle).ordered { |delivery_info, properties, broker, e|
+          broker.requeue delivery_info.delivery_tag
+          true
+        }
+        allow(worker).to receive(:error_acknowledgements).and_return([requeuer])
+        expect(broker).to_not receive(:ack)
+        expect(broker).to_not receive(:nack)
+        expect(broker).to receive(:requeue)
+
+        worker.handle_message(consumer, delivery_info, properties, payload)
+      end
+    end
+
+
     context 'when the consumer raises an exception' do
       before { allow(consumer_instance).to receive(:process).and_raise('a consumer error') }
 
@@ -94,5 +113,40 @@ describe Hutch::Worker do
       end
     end
   end
+
+
+  describe '#acknowledge_error' do
+    let(:delivery_info) { double('Delivery Info', routing_key: '',
+                                 delivery_tag: 'dt') }
+    let(:properties) { double('Properties', message_id: 'abc123') }
+
+    subject { worker.acknowledge_error delivery_info, properties, broker, StandardError.new }
+
+    it 'stops when it runs a successful acknowledgement' do
+      skip_ack = double handle: false
+      always_ack = double handle: true
+      never_used = double handle: true
+
+      allow(worker).
+        to receive(:error_acknowledgements).
+        and_return([skip_ack, always_ack, never_used])
+
+      expect(never_used).to_not receive(:handle)
+
+      subject
+    end
+
+    it 'defaults to nacking' do
+      skip_ack = double handle: false
+
+      allow(worker).
+        to receive(:error_acknowledgements).
+        and_return([skip_ack, skip_ack])
+
+      expect(broker).to receive(:nack)
+
+      subject
+    end
+  end
 end
 

data/spec/spec_helper.rb

@@ -16,6 +16,12 @@ require 'logger'
 RSpec.configure do |config|
   config.before(:all) { Hutch::Config.log_level = Logger::FATAL }
   config.raise_errors_for_deprecations!
+
+  if defined?(JRUBY_VERSION)
+    config.filter_run_excluding adapter: :bunny
+  else
+    config.filter_run_excluding adapter: :march_hare
+  end
 end
 
 # Constants (classes, etc) defined within a block passed to this method
@@ -32,4 +38,3 @@ end
 def deep_copy(obj)
   Marshal.load(Marshal.dump(obj))
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dkastner-hutch
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.19.0.pre1
 platform: ruby
 authors:
 - Harry Marr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -117,6 +117,10 @@ files:
 - examples/producer.rb
 - hutch.gemspec
 - lib/hutch.rb
+- lib/hutch/acknowledgements/nack_on_all_failures.rb
+- lib/hutch/adapter.rb
+- lib/hutch/adapters/bunny.rb
+- lib/hutch/adapters/march_hare.rb
 - lib/hutch/broker.rb
 - lib/hutch/cli.rb
 - lib/hutch/config.rb
@@ -162,9 +166,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2