hutch 0.24.0 → 1.0.0

data/bin/ci/before_build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+CTL=${BUNNY_RABBITMQCTL:-"sudo rabbitmqctl"}
+PLUGINS=${BUNNY_RABBITMQ_PLUGINS:-"sudo rabbitmq-plugins"}
+
+echo "Will use rabbitmqctl at ${CTL}"
+echo "Will use rabbitmq-plugins at ${PLUGINS}"
+
+$PLUGINS enable rabbitmq_management
+
+sleep 3
+
+# guest:guest has full access to /
+$CTL add_vhost /
+$CTL add_user guest guest
+$CTL set_permissions -p / guest ".*" ".*" ".*"
+
+# Reduce retention policy for faster publishing of stats
+$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management,       sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().' || true
+$CTL eval  'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().' || true

data/bin/ci/install_on_debian.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+sudo apt-get install -y wget
+wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
+
+sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list <<EOF
+deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang
+deb https://dl.bintray.com/rabbitmq/debian bionic main
+EOF
+
+sudo apt-get update -y
+sudo apt-get upgrade -y
+sudo apt-get install -y rabbitmq-server
+
+sudo service rabbitmq-server start
+
+until sudo lsof -i:5672; do echo "Waiting for RabbitMQ to start..."; sleep 1; done

data/hutch.gemspec

@@ -3,21 +3,21 @@ require File.expand_path('../lib/hutch/version', __FILE__)
 Gem::Specification.new do |gem|
   if defined?(JRUBY_VERSION)
     gem.platform = 'java'
-    gem.add_runtime_dependency 'march_hare', '>= 2.22.0'
+    gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
   else
     gem.platform = Gem::Platform::RUBY
-    gem.add_runtime_dependency 'bunny', '>= 2.6.3'
+    gem.add_runtime_dependency 'bunny', '>= 2.15', '< 2.16'
   end
   gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
-  gem.add_runtime_dependency 'multi_json', '~> 1.12'
-  gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 6'
+  gem.add_runtime_dependency 'multi_json', '~> 1.14'
+  gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 7'
 
   gem.name = 'hutch'
   gem.summary = 'Easy inter-service communication using RabbitMQ.'
   gem.description = 'Hutch is a Ruby library for enabling asynchronous ' \
                     'inter-service communication using RabbitMQ.'
   gem.version = Hutch::VERSION.dup
-  gem.required_ruby_version = '>= 2.0'
+  gem.required_ruby_version = '>= 2.2'
   gem.authors = ['Harry Marr']
   gem.email = ['developers@gocardless.com']
   gem.homepage = 'https://github.com/gocardless/hutch'

data/lib/hutch.rb

@@ -14,6 +14,8 @@ require 'hutch/exceptions'
 require 'hutch/tracers'
 
 module Hutch
+  @@connection_mutex = Mutex.new
+
   def self.register_consumer(consumer)
     self.consumers << consumer
   end
@@ -40,10 +42,12 @@ module Hutch
   # @param config [Hash] Configuration
   # @option options [Boolean] :enable_http_api_use
   def self.connect(options = {}, config = Hutch::Config)
-    return if connected?
-
-    @broker = Hutch::Broker.new(config)
-    @broker.connect(options)
+    @@connection_mutex.synchronize do
+      unless connected?
+        @broker = Hutch::Broker.new(config)
+        @broker.connect(options)
+      end
+    end
   end
 
   def self.disconnect

data/lib/hutch/broker.rb

@@ -11,6 +11,24 @@ module Hutch
 
     attr_accessor :connection, :channel, :exchange, :api_client
 
+
+    DEFAULT_AMQP_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
+      when "ruby" then
+        AMQ::Protocol::DEFAULT_PORT
+      end
+
+    DEFAULT_AMQPS_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
+      when "ruby" then
+        AMQ::Protocol::TLS_PORT
+      end
+
+
     # @param config [nil,Hash] Configuration override
     def initialize(config = nil)
       @config = config || Hutch::Config
@@ -104,11 +122,12 @@ module Hutch
 
     def declare_exchange(ch = channel)
       exchange_name = @config[:mq_exchange]
+      exchange_type = @config[:mq_exchange_type]
       exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
       logger.info "using topic exchange '#{exchange_name}'"
 
       with_bunny_precondition_handler('exchange') do
-        ch.topic(exchange_name, exchange_options)
+        Bunny::Exchange.new(ch, exchange_type, exchange_name, exchange_options)
       end
     end
 
@@ -163,12 +182,15 @@ module Hutch
     # Return a mapping of queue names to the routing keys they're bound to.
     def bindings
       results = Hash.new { |hash, key| hash[key] = [] }
-      api_client.bindings.each do |binding|
-        next if binding['destination'] == binding['routing_key']
-        next unless binding['source'] == @config[:mq_exchange]
-        next unless binding['vhost'] == @config[:mq_vhost]
+
+      filtered = api_client.bindings.
+        reject { |b| b['destination'] == b['routing_key'] }.
+        select { |b| b['source'] == @config[:mq_exchange] && b['vhost'] == @config[:mq_vhost] }
+
+      filtered.each do |binding|
         results[binding['destination']] << binding['routing_key']
       end
+
       results
     end
 
@@ -176,8 +198,8 @@ module Hutch
     def unbind_redundant_bindings(queue, routing_keys)
       return unless http_api_use_enabled?
 
-      bindings.each do |dest, keys|
-        next unless dest == queue.name
+      filtered = bindings.select { |dest, keys| dest == queue.name }
+      filtered.each do |dest, keys|
         keys.reject { |key| routing_keys.include?(key) }.each do |key|
           logger.debug "removing redundant binding #{queue.name} <--> #{key}"
           queue.unbind(exchange, routing_key: key)
@@ -280,8 +302,8 @@ module Hutch
         params[:write_timeout]      = @config[:write_timeout]
 
 
-        params[:automatically_recover] = true
-        params[:network_recovery_interval] = 1
+        params[:automatically_recover] = @config[:automatically_recover]
+        params[:network_recovery_interval] = @config[:network_recovery_interval]
 
         params[:logger] = @config[:client_logger] if @config[:client_logger]
       end
@@ -292,13 +314,18 @@ module Hutch
 
       u = URI.parse(@config[:uri])
 
+      @config[:mq_tls]      = u.scheme == 'amqps'
       @config[:mq_host]     = u.host
-      @config[:mq_port]     = u.port
+      @config[:mq_port]     = u.port || default_mq_port
       @config[:mq_vhost]    = u.path.sub(/^\//, "")
       @config[:mq_username] = u.user
       @config[:mq_password] = u.password
     end
 
+    def default_mq_port
+      @config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
+    end
+
     def sanitized_uri
       p = connection_params
       scheme = p[:tls] ? "amqps" : "amqp"

data/lib/hutch/cli.rb

@@ -33,6 +33,20 @@ module Hutch
     def load_app
       # Try to load a Rails app in the current directory
       load_rails_app('.') if Hutch::Config.autoload_rails
+      set_up_code_paths!
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def set_up_code_paths!
       Hutch::Config.require_paths.each do |path|
         # See if each path is a Rails app. If so, try to load it.
         next if load_rails_app(path)
@@ -51,16 +65,6 @@ module Hutch
           $LOAD_PATH.pop
         end
       end
-
-      # Because of the order things are required when we run the Hutch binary
-      # in hutch/bin, the Sentry Raven gem gets required **after** the error
-      # handlers are set up. Due to this, we never got any Sentry notifications
-      # when an error occurred in any of the consumers.
-      if defined?(Raven)
-        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
-      end
-
-      true
     end
 
     def load_rails_app(path)
@@ -72,8 +76,8 @@ module Hutch
         end
         rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
         if is_rails_app && File.exist?(rails_path)
-          logger.info "found rails project (#{path}), booting app"
           ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
           require rails_path
           ::Rails.application.eager_load!
           return true
@@ -90,6 +94,9 @@ module Hutch
       @worker.run
       :success
     rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle_setup_exception(ex)
+      end
       logger.fatal ex.message
       :error
     end
@@ -189,6 +196,10 @@ module Hutch
           Hutch::Config.pidfile = pidfile
         end
 
+        opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
+          Hutch::Config.group = group
+        end
+
         opts.on('--version', 'Print the version and exit') do
           puts "hutch v#{VERSION}"
           exit 0

data/lib/hutch/config.rb

@@ -49,6 +49,9 @@ module Hutch
     # RabbitMQ Exchange to use for publishing
     string_setting :mq_exchange, 'hutch'
 
+    # RabbitMQ Exchange type to use for publishing
+    string_setting :mq_exchange_type, 'topic'
+
     # RabbitMQ vhost to use
     string_setting :mq_vhost, '/'
 
@@ -89,6 +92,12 @@ module Hutch
     # Bunny's socket write timeout
     number_setting :write_timeout, 11
 
+    # Bunny's enable/disable network recovery
+    boolean_setting :automatically_recover, true
+
+    # Bunny's reconnect interval
+    number_setting :network_recovery_interval, 1
+
     # FIXME: DOCUMENT THIS
     number_setting :graceful_exit_timeout, 11
 
@@ -136,6 +145,8 @@ module Hutch
     # Prefix displayed on the consumers tags.
     string_setting :consumer_tag_prefix, 'hutch'
 
+    string_setting :group, ''
+
     # Set of all setting keys
     ALL_KEYS = @boolean_keys + @number_keys + @string_keys
 
@@ -167,6 +178,7 @@ module Hutch
         # that will fall back to "nack unconditionally"
         error_acknowledgements: [],
         setup_procs: [],
+        consumer_groups: {},
         tracer: Hutch::Tracers::NullTracer,
         namespace: nil,
         pidfile: nil,

data/lib/hutch/consumer.rb

@@ -29,14 +29,38 @@ module Hutch
       # wants to subscribe to.
       def consume(*routing_keys)
         @routing_keys = self.routing_keys.union(routing_keys)
+        # these are opt-in
+        @queue_mode = nil
+        @queue_type = nil
       end
 
+      attr_reader :queue_mode, :queue_type, :initial_group_size
+
       # Explicitly set the queue name
       def queue_name(name)
         @queue_name = name
       end
 
-      # Allow to specify custom arguments that will be passed when creating the queue.
+      # Explicitly set the queue mode to 'lazy'
+      def lazy_queue
+        @queue_mode = 'lazy'
+      end
+
+      # Explicitly set the queue type to 'classic'
+      def classic_queue
+        @queue_type = 'classic'
+      end
+
+      # Explicitly set the queue type to 'quorum'
+      # @param [Hash] options the options params related to quorum queue
+      # @option options [Integer] :initial_group_size Initial Replication Factor
+      def quorum_queue(options = {})
+        @queue_type = 'quorum'
+        @initial_group_size = options[:initial_group_size]
+      end
+
+      # Configures an optional argument that will be passed when declaring the queue.
+      # Prefer using a policy to this DSL: https://www.rabbitmq.com/parameters.html#policies
       def arguments(arguments = {})
         @arguments = arguments
       end
@@ -58,7 +82,13 @@ module Hutch
 
       # Returns consumer custom arguments.
       def get_arguments
-        @arguments || {}
+        all_arguments = @arguments || {}
+
+        all_arguments['x-queue-mode'] = @queue_mode if @queue_mode
+        all_arguments['x-queue-type'] = @queue_type if @queue_type
+        all_arguments['x-quorum-initial-group-size'] = @initial_group_size if @initial_group_size
+
+        all_arguments
       end
 
       # Accessor for the consumer's routing key.

data/lib/hutch/error_handlers.rb

@@ -4,6 +4,6 @@ module Hutch
     autoload :Sentry,      'hutch/error_handlers/sentry'
     autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
     autoload :Airbrake,    'hutch/error_handlers/airbrake'
-    autoload :Opbeat,      'hutch/error_handlers/opbeat'
+    autoload :Rollbar,     'hutch/error_handlers/rollbar'
   end
 end

data/lib/hutch/error_handlers/airbrake.rb

@@ -1,10 +1,10 @@
 require 'hutch/logging'
 require 'airbrake'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Airbrake
-      include Logging
+    class Airbrake < Base
 
       def handle(properties, payload, consumer, ex)
         message_id = properties.message_id
@@ -31,6 +31,24 @@ module Hutch
           })
         end
       end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Airbrake"
+        logger.error "#{ex.class} - #{ex.message}"
+
+        if ::Airbrake.respond_to?(:notify_or_ignore)
+          ::Airbrake.notify_or_ignore(ex, {
+            error_class: ex.class.name,
+            error_message: "#{ ex.class.name }: #{ ex.message }",
+            backtrace: ex.backtrace,
+            cgi_data: ENV.to_hash,
+          })
+        else
+          ::Airbrake.notify(ex, {
+            cgi_data: ENV.to_hash,
+          })
+        end
+      end
     end
   end
 end

data/lib/hutch/error_handlers/base.rb

@@ -0,0 +1,15 @@
+module Hutch
+  module ErrorHandlers
+    class Base
+      include Logging
+
+      def handle(properties, payload, consumer, ex)
+        raise NotImplementedError.new
+      end
+
+      def handle_setup_exception(ex)
+        raise NotImplementedError.new
+      end
+    end
+  end
+end

data/lib/hutch/error_handlers/honeybadger.rb

@@ -1,28 +1,42 @@
 require 'hutch/logging'
 require 'honeybadger'
+require 'hutch/error_handlers/base'
 
 module Hutch
   module ErrorHandlers
-    class Honeybadger
-      include Logging
+    # Error handler for the Honeybadger.io service
+    class Honeybadger < Base
 
       def handle(properties, payload, consumer, ex)
         message_id = properties.message_id
         prefix = "message(#{message_id || '-'}):"
         logger.error "#{prefix} Logging event to Honeybadger"
         logger.error "#{prefix} #{ex.class} - #{ex.message}"
-        ::Honeybadger.notify_or_ignore(
-          :error_class => ex.class.name,
-          :error_message => "#{ ex.class.name }: #{ ex.message }",
-          :backtrace => ex.backtrace,
-          :context => {
-            :message_id => message_id,
-            :consumer => consumer
-          },
-          :parameters => {
-            :payload => payload
-          }
-        )
+        notify_honeybadger(error_class: ex.class.name,
+                           error_message: "#{ex.class.name}: #{ex.message}",
+                           backtrace: ex.backtrace,
+                           context: { message_id: message_id,
+                                      consumer: consumer },
+                           parameters: { payload: payload })
+      end
+
+      def handle_setup_exception(ex)
+        logger.error "Logging setup exception to Honeybadger"
+        logger.error "#{ex.class} - #{ex.message}"
+        notify_honeybadger(error_class: ex.class.name,
+                           error_message: "#{ex.class.name}: #{ex.message}",
+                           backtrace: ex.backtrace)
+      end
+
+      # Wrap API to support 3.0.0+
+      #
+      # @see https://github.com/honeybadger-io/honeybadger-ruby/blob/master/CHANGELOG.md#300---2017-02-06
+      def notify_honeybadger(message)
+        if ::Honeybadger.respond_to?(:notify_or_ignore)
+          ::Honeybadger.notify_or_ignore(message)
+        else
+          ::Honeybadger.notify(message)
+        end
       end
     end
   end