hutch 0.27.0 → 0.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07e4ec8d85eefed7b25ae9bb43fb3c0ff8ffe0a4f24526b986686906aaa0f5cc
-  data.tar.gz: 94062cdf67c00e9224d3fee9be1fa294ba0e234f8400ec4efc640eaecc07a2df
+  metadata.gz: 587ca7c6e7515a0efcf8999a5ad3bfc1ef186a701a758260f68abb528641b7cd
+  data.tar.gz: 7f4353919efb856a7ef9af3dc71e7d2024aaa85c1183da3a191bbbf2c22a8492
 SHA512:
-  metadata.gz: e5a9ef9a749741aee734246701cde0a99f1c36f02e730eb1e888d2a125dba6e4ed9f9cdf945656ebbf8dcd118775c4717110c4d542b6547df84119baeb29f0a4
-  data.tar.gz: 8a929dafd7c168256da9c349e4bf60a052456f2d34db4e717e3c46a08e461dac60f95ea6903a76a54bb934fbbcabb62a1c2e24b662469d3e5afd9e459a165b2c
+  metadata.gz: 9c00226f20383fc7ec3503197783a48e0d8f153dd7e45ea7d89dda7631f2e527f71b6d15182be677d24818e1ce1cfcf6766765ad6a0e67b753b76cc145b46782
+  data.tar.gz: 530af1729a20c10d555bac21529d18c94cb07b1957ce2b6c5386adede9e400c7f7f400491c7c716378fd4fc8d8a955d2537e956dc6d6994aab5fc3cd2f2b5970

data/.travis.yml

@@ -1,3 +1,6 @@
+dist: bionic
+sudo: required
+
 language: ruby
 cache: bundler
 before_install:
@@ -5,18 +8,16 @@ before_install:
 before_script:
   - "./bin/ci/install_on_debian.sh"
   - until sudo lsof -i:5672; do echo "Waiting for RabbitMQ to start..."; sleep 1; done
+  - "./bin/ci/before_build.sh"
 matrix:
   include:
     - rvm: "2.6.4"
     - rvm: "2.5.6"
     - rvm: "2.4.7"
     - rvm: "2.3.8"
-    - rvm: "jruby-9.2.8.0"
+    - rvm: "jruby-9.2.9.0"
     - rvm: "ruby-head"
   allow_failures:
     rvm:
-      - "jruby-9.2.8.0"
+      - "jruby-9.2.9.0"
       - ruby-head
-
-services:
-  - rabbitmq

data/CHANGELOG.md

@@ -1,7 +1,34 @@
 ## 0.28.0 (under development)
 
-No chages yet.
+### Enhancements
+
+  * Add lazy and quorum options for queues.
+
+    GitHub issue: [gocardless/hutch#341](https://github.com/gocardless/hutch/pull/341)
+
+    Contributed by: Arthur Del Esposte
+
+  * Log level in the message publisher switched to DEBUG.
+
+    GitHub issue: [gocardless/hutch#343](https://github.com/gocardless/hutch/pull/343)
+
+    Contributed by: Codruț Constantin Gușoi
+
+### Documentation
+
+  * Add zeitwerk note to README.
+
+    GitHub issue: [gocardless/hutch#342](https://github.com/gocardless/hutch/pull/342)
+
+    Contributed by: Paolo Zaccagnini
+
+### CI
+
+  * Use jruby-9.2.9.0
+
+    GitHub issue: [gocardless/hutch#336](https://github.com/gocardless/hutch/pull/336)
 
+    Contributed by: Olle Jonsson
 
 ## 0.27.0 (September 9th, 2019)
 

data/Gemfile

@@ -23,7 +23,7 @@ group :development, :test do
   gem "honeybadger"
   gem "coveralls", "~> 0.8.15", require: false
   gem "newrelic_rpm"
-  gem "airbrake", "~> 9.0"
+  gem "airbrake", "~> 10.0"
   gem "rollbar"
 end
 

data/README.md

@@ -109,6 +109,28 @@ class FailedPaymentConsumer
 end
 ```
 
+It is possible to set some custom options to consumer's queue explicitly.
+This example sets the consumer's queue as a
+[quorum queue](https://www.rabbitmq.com/quorum-queues.html)
+and to operate in the [lazy mode](https://www.rabbitmq.com/lazy-queues.html).
+The `initial_group_size`
+[argument](https://www.rabbitmq.com/quorum-queues.html#replication-factor) is
+optional.
+
+```ruby
+class FailedPaymentConsumer
+  include Hutch::Consumer
+  consume 'gc.ps.payment.failed'
+  queue_name 'failed_payments'
+  lazy_queue
+  quorum_queue initial_group_size: 3
+
+  def process(message)
+    mark_payment_as_failed(message[:id])
+  end
+end
+```
+
 You can also set custom arguments per consumer. This example declares a consumer with
 a maximum length of 10 messages:
 
@@ -219,6 +241,24 @@ directory of a Rails app, or pass the path to a Rails app in with the
 the `app/consumers/` directory, to allow them to be auto-loaded when Rails
 boots.
 
+If you're using the new Zeitwerk autoloader (enabled by default in Rails 6)
+and the consumers are not loaded in development environment you will need to
+trigger the autoloading in an initializer with
+
+```ruby
+::Zeitwerk::Loader.eager_load_all 
+```
+
+or with something more specific like
+
+```ruby
+autoloader = Rails.autoloaders.main
+
+Dir.glob(File.join('app/consumers', '*_consumer.rb')).each do |consumer|
+  autoloader.preload(consumer)
+end
+```
+
 To require files that define consumers manually, simply pass each file as an
 option to `--require`. Hutch will automatically detect whether you've provided
 a Rails app or a standard file, and take the appropriate behaviour:

data/bin/ci/before_build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+CTL=${BUNNY_RABBITMQCTL:-"sudo rabbitmqctl"}
+PLUGINS=${BUNNY_RABBITMQ_PLUGINS:-"sudo rabbitmq-plugins"}
+
+echo "Will use rabbitmqctl at ${CTL}"
+echo "Will use rabbitmq-plugins at ${PLUGINS}"
+
+$PLUGINS enable rabbitmq_management
+
+sleep 3
+
+# guest:guest has full access to /
+$CTL add_vhost /
+$CTL add_user guest guest
+$CTL set_permissions -p / guest ".*" ".*" ".*"
+
+# Reduce retention policy for faster publishing of stats
+$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management,       sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().' || true
+$CTL eval  'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().' || true

data/bin/ci/install_on_debian.sh

@@ -4,8 +4,8 @@ sudo apt-get install -y wget
 wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
 
 sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list <<EOF
-deb https://dl.bintray.com/rabbitmq-erlang/debian xenial erlang
-deb https://dl.bintray.com/rabbitmq/debian xenial main
+deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang
+deb https://dl.bintray.com/rabbitmq/debian bionic main
 EOF
 
 sudo apt-get update -y

data/hutch.gemspec

@@ -6,10 +6,10 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
   else
     gem.platform = Gem::Platform::RUBY
-    gem.add_runtime_dependency 'bunny', '>= 2.13', '< 2.15'
+    gem.add_runtime_dependency 'bunny', '>= 2.14', '< 2.16'
   end
   gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
-  gem.add_runtime_dependency 'multi_json', '~> 1.12'
+  gem.add_runtime_dependency 'multi_json', '~> 1.14'
   gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 7'
 
   gem.name = 'hutch'

data/lib/hutch/broker.rb

@@ -181,12 +181,15 @@ module Hutch
     # Return a mapping of queue names to the routing keys they're bound to.
     def bindings
       results = Hash.new { |hash, key| hash[key] = [] }
-      api_client.bindings.each do |binding|
-        next if binding['destination'] == binding['routing_key']
-        next unless binding['source'] == @config[:mq_exchange]
-        next unless binding['vhost'] == @config[:mq_vhost]
+
+      filtered = api_client.bindings.
+        reject { |b| b['destination'] == b['routing_key'] }.
+        select { |b| b['source'] == @config[:mq_exchange] && b['vhost'] == @config[:mq_vhost] }
+
+      filtered.each do |binding|
         results[binding['destination']] << binding['routing_key']
       end
+
       results
     end
 
@@ -194,8 +197,8 @@ module Hutch
     def unbind_redundant_bindings(queue, routing_keys)
       return unless http_api_use_enabled?
 
-      bindings.each do |dest, keys|
-        next unless dest == queue.name
+      filtered = bindings.select { |dest, keys| dest == queue.name }
+      filtered.each do |dest, keys|
         keys.reject { |key| routing_keys.include?(key) }.each do |key|
           logger.debug "removing redundant binding #{queue.name} <--> #{key}"
           queue.unbind(exchange, routing_key: key)

data/lib/hutch/cli.rb

@@ -33,6 +33,20 @@ module Hutch
     def load_app
       # Try to load a Rails app in the current directory
       load_rails_app('.') if Hutch::Config.autoload_rails
+      set_up_code_paths!
+
+      # Because of the order things are required when we run the Hutch binary
+      # in hutch/bin, the Sentry Raven gem gets required **after** the error
+      # handlers are set up. Due to this, we never got any Sentry notifications
+      # when an error occurred in any of the consumers.
+      if defined?(Raven)
+        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
+      end
+
+      true
+    end
+
+    def set_up_code_paths!
       Hutch::Config.require_paths.each do |path|
         # See if each path is a Rails app. If so, try to load it.
         next if load_rails_app(path)
@@ -51,16 +65,6 @@ module Hutch
           $LOAD_PATH.pop
         end
       end
-
-      # Because of the order things are required when we run the Hutch binary
-      # in hutch/bin, the Sentry Raven gem gets required **after** the error
-      # handlers are set up. Due to this, we never got any Sentry notifications
-      # when an error occurred in any of the consumers.
-      if defined?(Raven)
-        Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
-      end
-
-      true
     end
 
     def load_rails_app(path)

data/lib/hutch/consumer.rb

@@ -29,13 +29,30 @@ module Hutch
       # wants to subscribe to.
       def consume(*routing_keys)
         @routing_keys = self.routing_keys.union(routing_keys)
+        @queue_mode = 'default'
+        @queue_type = 'classic'
       end
 
+      attr_reader :queue_mode, :queue_type, :initial_group_size
+
       # Explicitly set the queue name
       def queue_name(name)
         @queue_name = name
       end
 
+      # Explicitly set the queue mode to 'lazy'
+      def lazy_queue
+        @queue_mode = 'lazy'
+      end
+
+      # Explicitly set the queue type to 'quorum'
+      # @param [Hash] options the options params related to quorum queue
+      # @option options [Integer] :initial_group_size Initial Replication Factor
+      def quorum_queue(options = {})
+        @queue_type = 'quorum'
+        @initial_group_size = options[:initial_group_size]
+      end
+
       # Allow to specify custom arguments that will be passed when creating the queue.
       def arguments(arguments = {})
         @arguments = arguments
@@ -58,7 +75,13 @@ module Hutch
 
       # Returns consumer custom arguments.
       def get_arguments
-        @arguments || {}
+        all_arguments = @arguments || {}
+        all_arguments['x-queue-mode'] = @queue_mode
+        all_arguments['x-queue-type'] = @queue_type
+        if @initial_group_size
+          all_arguments['x-quorum-initial-group-size'] = @initial_group_size
+        end
+        all_arguments
       end
 
       # Accessor for the consumer's routing key.

data/lib/hutch/publisher.rb

@@ -42,7 +42,7 @@ module Hutch
     private
 
     def log_publication(serializer, payload, routing_key)
-      logger.info {
+      logger.debug {
         spec =
           if serializer.binary?
             "#{payload.bytesize} bytes message"

data/lib/hutch/version.rb

@@ -1,3 +1,3 @@
 module Hutch
-  VERSION = '0.27.0'.freeze
+  VERSION = '0.28.0'.freeze
 end

data/spec/hutch/consumer_spec.rb

@@ -28,6 +28,19 @@ describe Hutch::Consumer do
     ComplexConsumer
   end
 
+  let(:consumer_with_custom_queue_options) do
+    unless defined? ConsumerWithCustomQueueOptions
+      class ConsumerWithCustomQueueOptions
+        include Hutch::Consumer
+        consume 'hutch.test1'
+        arguments foo: :bar
+        lazy_queue
+        quorum_queue
+      end
+    end
+    ConsumerWithCustomQueueOptions
+  end
+
   describe 'module inclusion' do
     it 'registers the class as a consumer' do
       expect(Hutch).to receive(:register_consumer) do |klass|
@@ -71,6 +84,36 @@ describe Hutch::Consumer do
     end
   end
 
+  describe '.lazy_queue' do
+    it 'does not use lazy mode by default' do
+      expect(simple_consumer.queue_mode).to eq('default')
+    end
+
+    context 'when queue mode has been set explicitly to lazy' do
+      it 'sets queue mode to lazy' do
+        expect(consumer_with_custom_queue_options.queue_mode).to eq('lazy')
+      end
+    end
+  end
+
+  describe '.quorum_queue' do
+    it 'does not have quorum type by default' do
+      expect(simple_consumer.queue_type).to eq('classic')
+    end
+
+    context 'when queue type has been set explicitly to quorum' do
+      it 'sets queue type to quorum' do
+        expect(consumer_with_custom_queue_options.queue_type).to eq('quorum')
+      end
+
+      it 'accepts initial group size as an option' do
+        consumer = simple_consumer
+        expect { consumer.quorum_queue(initial_group_size: 3) }
+          .to change { consumer.initial_group_size }.to(3)
+      end
+    end
+  end
+
   describe '.arguments' do
     let(:args) { { foo: :bar} }
 
@@ -84,13 +127,36 @@ describe Hutch::Consumer do
   describe '.get_arguments' do
 
     context 'when defined' do
-      it { expect(complex_consumer.get_arguments).to eq(foo: :bar) }
+      it { expect(complex_consumer.get_arguments).to include(foo: :bar) }
     end
 
     context 'when not defined' do
-      it { expect(simple_consumer.get_arguments).to eq({}) }
+      it 'has the default values for queue custom options' do
+        expect(simple_consumer.get_arguments).to have_key('x-queue-mode')
+          .and have_key('x-queue-type')
+      end
     end
 
+    context 'when queue is lazy' do
+      it 'has the x-queue-mode argument set to lazy' do
+        expect(consumer_with_custom_queue_options.get_arguments['x-queue-mode'])
+          .to eq('lazy')
+      end
+    end
+
+    context "when queue's type is quorum" do
+      let(:arguments) { consumer_with_custom_queue_options.get_arguments }
+      it 'has the x-queue-type argument set to quorum' do
+        expect(arguments['x-queue-type']).to eq('quorum')
+        expect(arguments).to_not have_key('x-quorum-initial-group-size')
+      end
+
+      it 'has the x-quorum-initial-group-size argument set to quorum' do
+        consumer_with_custom_queue_options.quorum_queue(initial_group_size: 5)
+        expect(arguments['x-queue-type']).to eq('quorum')
+        expect(arguments['x-quorum-initial-group-size']).to eq(5)
+      end
+    end
   end
 
   describe '.get_queue_name' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hutch
 version: !ruby/object:Gem::Version
-  version: 0.27.0
+  version: 0.28.0
 platform: ruby
 authors:
 - Harry Marr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-09 00:00:00.000000000 Z
+date: 2020-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.13'
+        version: '2.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '2.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.13'
+        version: '2.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '2.16'
 - !ruby/object:Gem::Dependency
   name: carrot-top
   requirement: !ruby/object:Gem::Requirement
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -97,6 +97,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/ci/before_build.sh
 - bin/ci/install_on_debian.sh
 - bin/hutch
 - examples/consumer.rb