hutch 0.21.0 → 0.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85403719cdd5fb675886fc2dd6a830a544b3f8ea
-  data.tar.gz: ca485c1fe62a41867f634d46a45798570b3d441f
+  metadata.gz: a331137bca776d401e85057a0caa84922dda9d1d
+  data.tar.gz: c677873d07dfad4a6be14574d413e3870febb1bd
 SHA512:
-  metadata.gz: 0ff8d4d207f98acadd27035454a23aa50119076621b01b71fa21d342df02ff67574454f0b26374a60a2b04d941d1b170a6dfd420d12136abb404a17990ab5ea7
-  data.tar.gz: 62fef649721ee040dd1555310a6cd6338da4d00807ab5a6f41cb940e131df919a97e3dc173b2c2a4a9bfff311bce5835f1d37652472229470353c7787af407f7
+  metadata.gz: dd21d047618cf83c463b7064042270d2886d7cc232c9d1234b542912e37e7250f8a700fb47fcdc62f31ab61789a4fc3f89f3946387187081d0c705f0024d0ffe
+  data.tar.gz: bee9c047725370823d44a5a392e1544559998d70c36b7b084d6ba7f8cf93907f822d2ce935c024d51a3c4cfb567d1a68ab402bde62b30f497a67d4df85edac43

data/.gitignore

@@ -5,3 +5,5 @@ Gemfile.lock
 .ruby-gemset
 .idea
 coverage
+.yardoc
+doc

data/.yardopts

@@ -0,0 +1,5 @@
+--load "./lib/yard-settings/yard-settings.rb"
+--title "Hutch API docs"
+-
+README.md
+

data/CHANGELOG.md

@@ -1,4 +1,58 @@
-## 0.21.0 — (unreleased)
+## 0.23.0 — (unreleased)
+
+No changes yet.
+
+
+## 0.22.0 — June 7th, 2016
+
+### Message Payload is Reported to Sentry
+
+Contributed by Matt Thompson.
+
+### Daemonization Flag Ignored on JRuby
+
+Hutch will no longer try to daemonize its process on JRuby
+(since it is not supported) and will emit a warning instead.
+
+Contributed by Olle Jonsson.
+
+### Custom Setup Steps in Hutch::Worker
+
+`Hutch::Worker` now accepts a list of callables that are invoked
+after queue setup.
+
+Contributed by Kelly Stannard.
+
+### More Flexible and Better Abstracted Hutch::Broker
+
+`Hutch::Broker` was refactored with some bits extracted into separate
+classes or methods, making them easier to override.
+
+Contributed by Aleksandar Ivanov and Ryan Hosford.
+
+### Configurable Consumer Thread Pool Exception Handling (MRI only)
+
+`:consumer_pool_abort_on_exception` is a new option
+(defaults to `false`) which defines whether Bunny's
+consumer work pool threads should abort on exception.
+The option is ignored on JRuby.
+
+Contributed by Seamus Abshere.
+
+### Worker: Log received messages using level DEBUG instead of INFO
+
+Received messages used to be logged using severity level INFO. 
+This has been lowered to DEBUG.
+
+Contributed by Jesper Josefsson.
+
+### Refactoring
+
+Olle Jonsson and Kelly Stannard have contributed
+multiple internal improvements that have no behaviour changes.
+
+
+## 0.21.0 — February 7th, 2016
 
 ### JRuby Compatibility Restored
 

data/Gemfile

@@ -13,7 +13,7 @@ group :development, :test do
   gem "honeybadger"
   gem "coveralls", require: false
   gem "newrelic_rpm"
-  gem "airbrake"
+  gem "airbrake", "~> 4.0"
 end
 
 group :development, :darwin do

data/README.md

@@ -11,7 +11,7 @@ in a service-oriented architecture, using RabbitMQ.
 To install with RubyGems:
 
 ```
-$ gem install hutch
+gem install hutch
 ```
 
 ## Project Maturity
@@ -107,7 +107,7 @@ a [Logger object](http://ruby-doc.org/stdlib-2.1.2/libdoc/logger/rdoc/Logger.htm
 class FailedPaymentConsumer
   include Hutch::Consumer
   consume 'gc.ps.payment.failed'
-  
+
   def process(message)
     logger.info "Marking payment #{message[:id]} as failed"
     mark_payment_as_failed(message[:id])

data/Rakefile

@@ -11,4 +11,11 @@ RSpec::Core::RakeTask.new(:spec) do |t|
   t.rspec_opts = %w(--color --format doc)
 end
 
-task :default => :spec
+task default: :spec
+
+#
+# Re-generate API docs
+#
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new

data/hutch.gemspec

@@ -6,13 +6,16 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency 'march_hare', '>= 2.16.0'
   else
     gem.platform = Gem::Platform::RUBY
-    gem.add_runtime_dependency 'bunny', '>= 2.2.2'
+    gem.add_runtime_dependency 'bunny', '>= 2.3.1'
   end
   gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
   gem.add_runtime_dependency 'multi_json', '~> 1.11.2'
   gem.add_runtime_dependency 'activesupport', '>= 3.0'
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'simplecov', '~> 0.7.1'
+  gem.add_development_dependency 'yard', '~> 0.8'
+  gem.add_development_dependency 'redcarpet', '> 0'
+  gem.add_development_dependency 'github-markup', '> 0'
 
   gem.name = 'hutch'
   gem.summary = 'Easy inter-service communication using RabbitMQ.'

data/lib/hutch.rb

@@ -14,7 +14,6 @@ require 'hutch/exceptions'
 require 'hutch/tracers'
 
 module Hutch
-
   def self.register_consumer(consumer)
     self.consumers << consumer
   end
@@ -35,11 +34,16 @@ module Hutch
     @global_properties ||= {}
   end
 
+  # Connects to broker, if not yet connected.
+  #
+  # @param options [Hash] Connection options
+  # @param config [Hash] Configuration
+  # @option options [Boolean] :enable_http_api_use
   def self.connect(options = {}, config = Hutch::Config)
-    unless connected?
-      @broker = Hutch::Broker.new(config)
-      @broker.connect(options)
-    end
+    return if connected?
+
+    @broker = Hutch::Broker.new(config)
+    @broker.connect(options)
   end
 
   def self.disconnect
@@ -50,10 +54,9 @@ module Hutch
     @broker
   end
 
+  # @return [Boolean]
   def self.connected?
-    return false unless broker
-    return false unless broker.connection
-    broker.connection.open?
+    broker && broker.connection && broker.connection.open?
   end
 
   def self.publish(*args)

data/lib/hutch/adapters/march_hare.rb

@@ -25,7 +25,7 @@ module Hutch
         ch.prefetch = prefetch if prefetch
       end
 
-      def create_channel(n = nil, consumer_pool_size = 1)
+      def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
         @connection.create_channel(n)
       end
 

data/lib/hutch/broker.rb

@@ -1,7 +1,9 @@
+require 'active_support/core_ext/object/blank'
+
 require 'carrot-top'
-require 'securerandom'
 require 'hutch/logging'
 require 'hutch/exceptions'
+require 'hutch/publisher'
 
 module Hutch
   class Broker
@@ -9,10 +11,20 @@ module Hutch
 
     attr_accessor :connection, :channel, :exchange, :api_client
 
+    # @param config [nil,Hash] Configuration override
     def initialize(config = nil)
       @config = config || Hutch::Config
     end
 
+    # Connect to broker
+    #
+    # @example
+    #   Hutch::Broker.new.connect(enable_http_api_use: true) do
+    #     # will disconnect after this block
+    #   end
+    #
+    # @param [Hash] options The options to connect with
+    # @option options [Boolean] :enable_http_api_use
     def connect(options = {})
       @options = options
       set_up_amqp_connection
@@ -41,42 +53,44 @@ module Hutch
     def disconnect
       @channel.close    if @channel
       @connection.close if @connection
-      @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
+      @channel = nil
+      @connection = nil
+      @exchange = nil
+      @api_client = nil
     end
 
-    # Connect to RabbitMQ via AMQP. This sets up the main connection and
-    # channel we use for talking to RabbitMQ. It also ensures the existance of
-    # the exchange we'll be using.
+    # Connect to RabbitMQ via AMQP
+    #
+    # This sets up the main connection and channel we use for talking to
+    # RabbitMQ. It also ensures the existence of the exchange we'll be using.
     def set_up_amqp_connection
       open_connection!
       open_channel!
-
-      exchange_name = @config[:mq_exchange]
-      exchange_options = { durable: true }.merge @config[:mq_exchange_options]
-      logger.info "using topic exchange '#{exchange_name}'"
-
-      with_bunny_precondition_handler('exchange') do
-        @exchange = @channel.topic(exchange_name, exchange_options)
-      end
+      declare_exchange!
+      declare_publisher!
     end
 
-    def open_connection!
+    def open_connection
       logger.info "connecting to rabbitmq (#{sanitized_uri})"
 
-      @connection = Hutch::Adapter.new(connection_params)
+      connection = Hutch::Adapter.new(connection_params)
 
       with_bunny_connection_handler(sanitized_uri) do
-        @connection.start
+        connection.start
       end
 
       logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
-      @connection
+      connection
     end
 
-    def open_channel!
-      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
-      @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
-        @connection.prefetch_channel(ch, @config[:channel_prefetch])
+    def open_connection!
+      @connection = open_connection
+    end
+
+    def open_channel
+      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
+      connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
+        connection.prefetch_channel(ch, @config[:channel_prefetch])
         if @config[:publisher_confirms] || @config[:force_publisher_confirms]
           logger.info 'enabling publisher confirms'
           ch.confirm_select
@@ -84,6 +98,28 @@ module Hutch
       end
     end
 
+    def open_channel!
+      @channel = open_channel
+    end
+
+    def declare_exchange(ch = channel)
+      exchange_name = @config[:mq_exchange]
+      exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
+      logger.info "using topic exchange '#{exchange_name}'"
+
+      with_bunny_precondition_handler('exchange') do
+        ch.topic(exchange_name, exchange_options)
+      end
+    end
+
+    def declare_exchange!(*args)
+      @exchange = declare_exchange(*args)
+    end
+
+    def declare_publisher!
+      @publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
+    end
+
     # Set up the connection to the RabbitMQ management API. Unfortunately, this
     # is necessary to do a few things that are impossible over AMQP. E.g.
     # listing queues and bindings.
@@ -119,7 +155,7 @@ module Hutch
     def queue(name, arguments = {})
       with_bunny_precondition_handler('queue') do
         namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
-        name = name.prepend(namespace + ":") unless namespace.empty?
+        name = name.prepend(namespace + ":") if namespace.present?
         channel.queue(name, durable: true, arguments: arguments)
       end
     end
@@ -127,7 +163,7 @@ module Hutch
     # Return a mapping of queue names to the routing keys they're bound to.
     def bindings
       results = Hash.new { |hash, key| hash[key] = [] }
-      @api_client.bindings.each do |binding|
+      api_client.bindings.each do |binding|
         next if binding['destination'] == binding['routing_key']
         next unless binding['source'] == @config[:mq_exchange]
         next unless binding['vhost'] == @config[:mq_vhost]
@@ -136,37 +172,32 @@ module Hutch
       results
     end
 
+    # Find the existing bindings, and unbind any redundant bindings
+    def unbind_redundant_bindings(queue, routing_keys)
+      return unless http_api_use_enabled?
+
+      bindings.each do |dest, keys|
+        next unless dest == queue.name
+        keys.reject { |key| routing_keys.include?(key) }.each do |key|
+          logger.debug "removing redundant binding #{queue.name} <--> #{key}"
+          queue.unbind(exchange, routing_key: key)
+        end
+      end
+    end
+
     # Bind a queue to the broker's exchange on the routing keys provided. Any
     # existing bindings on the queue that aren't present in the array of
     # routing keys will be unbound.
     def bind_queue(queue, routing_keys)
-      if http_api_use_enabled?
-        # Find the existing bindings, and unbind any redundant bindings
-        queue_bindings = bindings.select { |dest, keys| dest == queue.name }
-        queue_bindings.each do |dest, keys|
-          keys.reject { |key| routing_keys.include?(key) }.each do |key|
-            logger.debug "removing redundant binding #{queue.name} <--> #{key}"
-            queue.unbind(@exchange, routing_key: key)
-          end
-        end
-      end
+      unbind_redundant_bindings(queue, routing_keys)
 
       # Ensure all the desired bindings are present
       routing_keys.each do |routing_key|
         logger.debug "creating binding #{queue.name} <--> #{routing_key}"
-        queue.bind(@exchange, routing_key: routing_key)
+        queue.bind(exchange, routing_key: routing_key)
       end
     end
 
-    # Each subscriber is run in a thread. This calls Thread#join on each of the
-    # subscriber threads.
-    def wait_on_threads(timeout)
-      # Thread#join returns nil when the timeout is hit. If any return nil,
-      # the threads didn't all join so we return false.
-      per_thread_timeout = timeout.to_f / work_pool_threads.length
-      work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
-    end
-
     def stop
       if defined?(JRUBY_VERSION)
         channel.close
@@ -181,78 +212,39 @@ module Hutch
     end
 
     def requeue(delivery_tag)
-      @channel.reject(delivery_tag, true)
+      channel.reject(delivery_tag, true)
     end
 
     def reject(delivery_tag, requeue=false)
-      @channel.reject(delivery_tag, requeue)
+      channel.reject(delivery_tag, requeue)
     end
 
     def ack(delivery_tag)
-      @channel.ack(delivery_tag, false)
+      channel.ack(delivery_tag, false)
     end
 
     def nack(delivery_tag)
-      @channel.nack(delivery_tag, false, false)
+      channel.nack(delivery_tag, false, false)
     end
 
-    def publish(routing_key, message, properties = {}, options = {})
-      ensure_connection!(routing_key, message)
-
-      serializer = options[:serializer] || @config[:serializer]
-
-      non_overridable_properties = {
-        routing_key:  routing_key,
-        timestamp:    @connection.current_timestamp,
-        content_type: serializer.content_type,
-      }
-      properties[:message_id]   ||= generate_id
-
-      payload = serializer.encode(message)
-      logger.info {
-        spec =
-          if serializer.binary?
-            "#{payload.bytesize} bytes message"
-          else
-            "message '#{payload}'"
-          end
-        "publishing #{spec} to #{routing_key}"
-      }
-
-      response = @exchange.publish(payload, {persistent: true}.
-        merge(properties).
-        merge(global_properties).
-        merge(non_overridable_properties))
-
-      channel.wait_for_confirms if @config[:force_publisher_confirms]
-      response
+    def publish(*args)
+      @publisher.publish(*args)
     end
 
     def confirm_select(*args)
-      @channel.confirm_select(*args)
+      channel.confirm_select(*args)
     end
 
     def wait_for_confirms
-      @channel.wait_for_confirms
+      channel.wait_for_confirms
     end
 
+    # @return [Boolean] True if channel is set up to use publisher confirmations.
     def using_publisher_confirmations?
-      @channel.using_publisher_confirmations?
+      channel.using_publisher_confirmations?
     end
 
     private
-
-    def raise_publish_error(reason, routing_key, message)
-      msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
-      logger.error(msg)
-      raise PublishError, msg
-    end
-
-    def ensure_connection!(routing_key, message)
-      raise_publish_error('no connection to broker', routing_key, message) unless @connection
-      raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
-    end
-
     def api_config
       @api_config ||= OpenStruct.new.tap do |config|
         config.host = @config[:mq_api_host]
@@ -271,11 +263,7 @@ module Hutch
       {}.tap do |params|
         params[:host]               = @config[:mq_host]
         params[:port]               = @config[:mq_port]
-        params[:vhost]              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
-                                        @config[:mq_vhost]
-                                      else
-                                        Hutch::Adapter::DEFAULT_VHOST
-                                      end
+        params[:vhost]              = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
         params[:username]           = @config[:mq_username]
         params[:password]           = @config[:mq_password]
         params[:tls]                = @config[:mq_tls]
@@ -299,7 +287,7 @@ module Hutch
     end
 
     def parse_uri
-      return unless @config[:uri] && !@config[:uri].empty?
+      return if @config[:uri].blank?
 
       u = URI.parse(@config[:uri])
 
@@ -351,25 +339,16 @@ module Hutch
       raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
     end
 
-    def work_pool_threads
-      channel_work_pool.threads || []
-    end
-
     def channel_work_pool
-      @channel.work_pool
+      channel.work_pool
     end
 
     def consumer_pool_size
       @config[:consumer_pool_size]
     end
 
-    def generate_id
-      SecureRandom.uuid
+    def consumer_pool_abort_on_exception
+      @config[:consumer_pool_abort_on_exception]
     end
-
-    def global_properties
-      Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
-    end
-
   end
 end