hutch 0.18.0 → 1.1.0

data/lib/hutch/adapters/march_hare.rb

@@ -25,7 +25,7 @@ module Hutch
         ch.prefetch = prefetch if prefetch
       end
 
-      def create_channel(n = nil, consumer_pool_size = 1)
+      def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
         @connection.create_channel(n)
       end
 

data/lib/hutch/broker.rb

@@ -1,7 +1,9 @@
+require 'active_support/core_ext/object/blank'
+
 require 'carrot-top'
-require 'securerandom'
 require 'hutch/logging'
 require 'hutch/exceptions'
+require 'hutch/publisher'
 
 module Hutch
   class Broker
@@ -9,10 +11,38 @@ module Hutch
 
     attr_accessor :connection, :channel, :exchange, :api_client
 
+
+    DEFAULT_AMQP_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
+      when "ruby" then
+        AMQ::Protocol::DEFAULT_PORT
+      end
+
+    DEFAULT_AMQPS_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
+      when "ruby" then
+        AMQ::Protocol::TLS_PORT
+      end
+
+
+    # @param config [nil,Hash] Configuration override
     def initialize(config = nil)
       @config = config || Hutch::Config
     end
 
+    # Connect to broker
+    #
+    # @example
+    #   Hutch::Broker.new.connect(enable_http_api_use: true) do
+    #     # will disconnect after this block
+    #   end
+    #
+    # @param [Hash] options The options to connect with
+    # @option options [Boolean] :enable_http_api_use
     def connect(options = {})
       @options = options
       set_up_amqp_connection
@@ -41,41 +71,44 @@ module Hutch
     def disconnect
       @channel.close    if @channel
       @connection.close if @connection
-      @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
+      @channel = nil
+      @connection = nil
+      @exchange = nil
+      @api_client = nil
     end
 
-    # Connect to RabbitMQ via AMQP. This sets up the main connection and
-    # channel we use for talking to RabbitMQ. It also ensures the existance of
-    # the exchange we'll be using.
+    # Connect to RabbitMQ via AMQP
+    #
+    # This sets up the main connection and channel we use for talking to
+    # RabbitMQ. It also ensures the existence of the exchange we'll be using.
     def set_up_amqp_connection
       open_connection!
       open_channel!
-
-      exchange_name = @config[:mq_exchange]
-      logger.info "using topic exchange '#{exchange_name}'"
-
-      with_bunny_precondition_handler('exchange') do
-        @exchange = @channel.topic(exchange_name, durable: true)
-      end
+      declare_exchange!
+      declare_publisher!
     end
 
-    def open_connection!
+    def open_connection
       logger.info "connecting to rabbitmq (#{sanitized_uri})"
 
-      @connection = Hutch::Adapter.new(connection_params)
+      connection = Hutch::Adapter.new(connection_params)
 
       with_bunny_connection_handler(sanitized_uri) do
-        @connection.start
+        connection.start
       end
 
       logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
-      @connection
+      connection
     end
 
-    def open_channel!
-      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
-      @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
-        @connection.prefetch_channel(ch, @config[:channel_prefetch])
+    def open_connection!
+      @connection = open_connection
+    end
+
+    def open_channel
+      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
+      connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
+        connection.prefetch_channel(ch, @config[:channel_prefetch])
         if @config[:publisher_confirms] || @config[:force_publisher_confirms]
           logger.info 'enabling publisher confirms'
           ch.confirm_select
@@ -83,6 +116,29 @@ module Hutch
       end
     end
 
+    def open_channel!
+      @channel = open_channel
+    end
+
+    def declare_exchange(ch = channel)
+      exchange_name = @config[:mq_exchange]
+      exchange_type = @config[:mq_exchange_type]
+      exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
+      logger.info "using topic exchange '#{exchange_name}'"
+
+      with_bunny_precondition_handler('exchange') do
+        Bunny::Exchange.new(ch, exchange_type, exchange_name, exchange_options)
+      end
+    end
+
+    def declare_exchange!(*args)
+      @exchange = declare_exchange(*args)
+    end
+
+    def declare_publisher!
+      @publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
+    end
+
     # Set up the connection to the RabbitMQ management API. Unfortunately, this
     # is necessary to do a few things that are impossible over AMQP. E.g.
     # listing queues and bindings.
@@ -118,7 +174,7 @@ module Hutch
     def queue(name, arguments = {})
       with_bunny_precondition_handler('queue') do
         namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
-        name = name.prepend(namespace + ":") unless namespace.empty?
+        name = name.prepend(namespace + ":") if namespace.present?
         channel.queue(name, durable: true, arguments: arguments)
       end
     end
@@ -126,46 +182,44 @@ module Hutch
     # Return a mapping of queue names to the routing keys they're bound to.
     def bindings
       results = Hash.new { |hash, key| hash[key] = [] }
-      @api_client.bindings.each do |binding|
-        next if binding['destination'] == binding['routing_key']
-        next unless binding['source'] == @config[:mq_exchange]
-        next unless binding['vhost'] == @config[:mq_vhost]
+
+      filtered = api_client.bindings.
+        reject { |b| b['destination'] == b['routing_key'] }.
+        select { |b| b['source'] == @config[:mq_exchange] && b['vhost'] == @config[:mq_vhost] }
+
+      filtered.each do |binding|
         results[binding['destination']] << binding['routing_key']
       end
+
       results
     end
 
+    # Find the existing bindings, and unbind any redundant bindings
+    def unbind_redundant_bindings(queue, routing_keys)
+      return unless http_api_use_enabled?
+
+      filtered = bindings.select { |dest, keys| dest == queue.name }
+      filtered.each do |dest, keys|
+        keys.reject { |key| routing_keys.include?(key) }.each do |key|
+          logger.debug "removing redundant binding #{queue.name} <--> #{key}"
+          queue.unbind(exchange, routing_key: key)
+        end
+      end
+    end
+
     # Bind a queue to the broker's exchange on the routing keys provided. Any
     # existing bindings on the queue that aren't present in the array of
     # routing keys will be unbound.
     def bind_queue(queue, routing_keys)
-      if http_api_use_enabled?
-        # Find the existing bindings, and unbind any redundant bindings
-        queue_bindings = bindings.select { |dest, keys| dest == queue.name }
-        queue_bindings.each do |dest, keys|
-          keys.reject { |key| routing_keys.include?(key) }.each do |key|
-            logger.debug "removing redundant binding #{queue.name} <--> #{key}"
-            queue.unbind(@exchange, routing_key: key)
-          end
-        end
-      end
+      unbind_redundant_bindings(queue, routing_keys)
 
       # Ensure all the desired bindings are present
       routing_keys.each do |routing_key|
         logger.debug "creating binding #{queue.name} <--> #{routing_key}"
-        queue.bind(@exchange, routing_key: routing_key)
+        queue.bind(exchange, routing_key: routing_key)
       end
     end
 
-    # Each subscriber is run in a thread. This calls Thread#join on each of the
-    # subscriber threads.
-    def wait_on_threads(timeout)
-      # Thread#join returns nil when the timeout is hit. If any return nil,
-      # the threads didn't all join so we return false.
-      per_thread_timeout = timeout.to_f / work_pool_threads.length
-      work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
-    end
-
     def stop
       if defined?(JRUBY_VERSION)
         channel.close
@@ -180,67 +234,40 @@ module Hutch
     end
 
     def requeue(delivery_tag)
-      @channel.reject(delivery_tag, true)
+      channel.reject(delivery_tag, true)
     end
 
     def reject(delivery_tag, requeue=false)
-      @channel.reject(delivery_tag, requeue)
+      channel.reject(delivery_tag, requeue)
     end
 
     def ack(delivery_tag)
-      @channel.ack(delivery_tag, false)
+      channel.ack(delivery_tag, false)
     end
 
     def nack(delivery_tag)
-      @channel.nack(delivery_tag, false, false)
+      channel.nack(delivery_tag, false, false)
     end
 
-    def publish(routing_key, message, properties = {})
-      ensure_connection!(routing_key, message)
-
-      non_overridable_properties = {
-        routing_key: routing_key,
-        timestamp: @connection.current_timestamp,
-        content_type: 'application/json'
-      }
-      properties[:message_id] ||= generate_id
-
-      json = JSON.dump(message)
-      logger.info("publishing message '#{json}' to #{routing_key}")
-      response = @exchange.publish(json, {persistent: true}.
-        merge(properties).
-        merge(global_properties).
-        merge(non_overridable_properties))
-
-      channel.wait_for_confirms if @config[:force_publisher_confirms]
-      response
+    def publish(*args)
+      @publisher.publish(*args)
     end
 
     def confirm_select(*args)
-      @channel.confirm_select(*args)
+      channel.confirm_select(*args)
     end
 
     def wait_for_confirms
-      @channel.wait_for_confirms
+      channel.wait_for_confirms
     end
 
+    # @return [Boolean] True if channel is set up to use publisher confirmations.
     def using_publisher_confirmations?
-      @channel.using_publisher_confirmations?
+      channel.using_publisher_confirmations?
     end
 
     private
 
-    def raise_publish_error(reason, routing_key, message)
-      msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
-      logger.error(msg)
-      raise PublishError, msg
-    end
-
-    def ensure_connection!(routing_key, message)
-      raise_publish_error('no connection to broker', routing_key, message) unless @connection
-      raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
-    end
-
     def api_config
       @api_config ||= OpenStruct.new.tap do |config|
         config.host = @config[:mq_api_host]
@@ -259,41 +286,46 @@ module Hutch
       {}.tap do |params|
         params[:host]               = @config[:mq_host]
         params[:port]               = @config[:mq_port]
-        params[:vhost]              = if @config[:mq_vhost] && "" != @config[:mq_vhost]
-                                        @config[:mq_vhost]
-                                      else
-                                        Hutch::Adapter::DEFAULT_VHOST
-                                      end
+        params[:vhost]              = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
         params[:username]           = @config[:mq_username]
         params[:password]           = @config[:mq_password]
         params[:tls]                = @config[:mq_tls]
         params[:tls_key]            = @config[:mq_tls_key]
         params[:tls_cert]           = @config[:mq_tls_cert]
+        params[:verify_peer]        = @config[:mq_verify_peer]
+        if @config[:mq_tls_ca_certificates]
+          params[:tls_ca_certificates] = @config[:mq_tls_ca_certificates]
+        end
         params[:heartbeat]          = @config[:heartbeat]
         params[:connection_timeout] = @config[:connection_timeout]
         params[:read_timeout]       = @config[:read_timeout]
         params[:write_timeout]      = @config[:write_timeout]
 
 
-        params[:automatically_recover] = true
-        params[:network_recovery_interval] = 1
+        params[:automatically_recover] = @config[:automatically_recover]
+        params[:network_recovery_interval] = @config[:network_recovery_interval]
 
-        params[:client_logger] = @config[:client_logger] if @config[:client_logger]
+        params[:logger] = @config[:client_logger] if @config[:client_logger]
       end
     end
 
     def parse_uri
-      return unless @config[:uri] && !@config[:uri].empty?
+      return if @config[:uri].blank?
 
       u = URI.parse(@config[:uri])
 
+      @config[:mq_tls]      = u.scheme == 'amqps'
       @config[:mq_host]     = u.host
-      @config[:mq_port]     = u.port
+      @config[:mq_port]     = u.port || default_mq_port
       @config[:mq_vhost]    = u.path.sub(/^\//, "")
       @config[:mq_username] = u.user
       @config[:mq_password] = u.password
     end
 
+    def default_mq_port
+      @config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
+    end
+
     def sanitized_uri
       p = connection_params
       scheme = p[:tls] ? "amqps" : "amqp"
@@ -335,24 +367,16 @@ module Hutch
       raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
     end
 
-    def work_pool_threads
-      channel_work_pool.threads || []
-    end
-
     def channel_work_pool
-      @channel.work_pool
+      channel.work_pool
     end
 
     def consumer_pool_size
       @config[:consumer_pool_size]
     end
 
-    def generate_id
-      SecureRandom.uuid
-    end
-
-    def global_properties
-      Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
+    def consumer_pool_abort_on_exception
+      @config[:consumer_pool_abort_on_exception]
     end
   end
 end

data/lib/hutch/cli.rb

@@ -1,9 +1,9 @@
 require 'optparse'
 
-require 'hutch/version'
 require 'hutch/logging'
-require 'hutch/exceptions'
 require 'hutch/config'
+require 'hutch/version'
+require 'hutch/exceptions'
 
 module Hutch
   class CLI
@@ -11,9 +11,10 @@ module Hutch
 
     # Run a Hutch worker with the command line interface.
     def run(argv = ARGV)
+      Hutch::Config.initialize
       parse_options(argv)
 
-      ::Process.daemon(true) if Hutch::Config.daemonise
+      daemonise_process
 
       write_pid if Hutch::Config.pidfile
 
@@ -32,6 +33,23 @@ module Hutch
     def load_app
       # Try to load a Rails app in the current directory
       load_rails_app('.') if Hutch::Config.autoload_rails
+      set_up_code_paths!
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::SentryRaven.new
+      end
+      if defined?(Sentry)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def set_up_code_paths!
       Hutch::Config.require_paths.each do |path|
         # See if each path is a Rails app. If so, try to load it.
         next if load_rails_app(path)
@@ -42,34 +60,27 @@ module Hutch
           # Need to add '.' to load path for relative requires
           $LOAD_PATH << '.'
           require path
-        rescue LoadError
-          logger.fatal "could not load file '#{path}'"
+        rescue LoadError => e
+          logger.fatal "could not load file '#{path}': #{e}"
           return false
         ensure
           # Clean up load path
           $LOAD_PATH.pop
         end
       end
-
-      # Because of the order things are required when we run the Hutch binary
-      # in hutch/bin, the Sentry Raven gem gets required **after** the error
-      # handlers are set up. Due to this, we never got any Sentry notifications
-      # when an error occurred in any of the consumers.
-      if defined?(Raven)
-        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
-      end
-
-      true
     end
 
     def load_rails_app(path)
       # path should point to the app's top level directory
       if File.directory?(path)
-        # Smells like a Rails app if it's got a config/environment.rb file
+        # Smells like a Rails app if it's got a script/rails or bin/rails file
+        is_rails_app = ['script/rails', 'bin/rails'].any? do |file|
+          File.exist?(File.expand_path(File.join(path, file)))
+        end
         rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
-        if File.exists?(rails_path)
-          logger.info "found rails project (#{path}), booting app"
+        if is_rails_app && File.exist?(rails_path)
           ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
+          logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
           require rails_path
           ::Rails.application.eager_load!
           return true
@@ -82,10 +93,13 @@ module Hutch
     # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
     def start_work_loop
       Hutch.connect
-      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
+      @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
       @worker.run
       :success
     rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
+      Hutch::Config[:error_handlers].each do |backend|
+        backend.handle_setup_exception(ex)
+      end
       logger.fatal ex.message
       :error
     end
@@ -106,14 +120,16 @@ module Hutch
           Hutch::Config.mq_tls = tls
         end
 
-        opts.on('--mq-tls-cert FILE', 'Certificate  for TLS client verification') do |file|
-          abort "Certificate file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_cert = file
+        opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
+          abort_without_file(file, 'Certificate file') do
+            Hutch::Config.mq_tls_cert = file
+          end
         end
 
         opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
-          abort "Private key file '#{file}' not found" unless File.exists?(file)
-          Hutch::Config.mq_tls_key = file
+          abort_without_file(file, 'Private key file') do
+            Hutch::Config.mq_tls_key = file
+          end
         end
 
         opts.on('--mq-exchange EXCHANGE',
@@ -151,7 +167,7 @@ module Hutch
           begin
             File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
           rescue Errno::ENOENT
-            abort "Config file '#{file}' not found"
+            abort_with_message("Config file '#{file}' not found")
           end
         end
 
@@ -183,6 +199,10 @@ module Hutch
           Hutch::Config.pidfile = pidfile
         end
 
+        opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
+          Hutch::Config.group = group
+        end
+
         opts.on('--version', 'Print the version and exit') do
           puts "hutch v#{VERSION}"
           exit 0
@@ -201,5 +221,26 @@ module Hutch
       File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
     end
 
+    private
+
+    def daemonise_process
+      return unless Hutch::Config.daemonise
+      if defined?(JRUBY_VERSION)
+        Hutch.logger.warn "JRuby ignores the --daemonise option"
+        return
+      end
+
+      ::Process.daemon(true)
+    end
+
+    def abort_without_file(file, file_description, &block)
+      abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
+
+      yield
+    end
+
+    def abort_with_message(message)
+      abort message
+    end
   end
 end