shoryuken 3.1.6 → 3.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3418f84df488f7c36c21d87b9fda96cedac66407
-  data.tar.gz: 0e1183b0fca34e62be3f9e965bbfcfc823111251
+  metadata.gz: 52a8705f1b474b12c77323776c903bd2e1ff6ba1
+  data.tar.gz: 25792ada1e4231fa80c7c461affa364bbe8f9afa
 SHA512:
-  metadata.gz: 0a028333d4c876ee617270d744eeddf59ef5005ac5937151734912dffcbbbb59798a05b9be4ae94b97bc1d991c35b01785c8985a99df341132a55ae63926389f
-  data.tar.gz: bfcdaf2ec179ddb46e21bffcea2413c7d196c0cacb8b81d7586e5f1645049eb5b1dc15aef86a520a04281118c65559d0849ecbf83b1780bb8a31d71c6f763027
+  metadata.gz: bcc31cc6ea76c07965f0ef5e4d13f29407fcf7423d8441df46867255cd5beaec0e694f9d5e45f6be181d1f1f926f51752f64d6027be8dfacfbd721052138de70
+  data.tar.gz: 4c7a4b04713c37edbf2e1e2965b3135a6d603ff686f42f74a45f6e5b1b9212f71db155703c92433785357281568ec705cd02e23413abcb46dc079b43df57bad7

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 rvm:
   - 2.0.0
-  - 2.1.0
+  - 2.1.10
   - 2.2.0
   - 2.3.3
   - 2.4.1

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [v3.1.7] - 2017-07-31
+
+- Allow polling strategy per group
+ - [#417](https://github.com/phstc/shoryuken/pull/417)
+
+- Add support for creating FIFO queues
+ - [#419](https://github.com/phstc/shoryuken/pull/419)
+
+- Allow receive message options per queue
+ - [#420](https://github.com/phstc/shoryuken/pull/420)
+
 ## [v3.1.6] - 2017-07-24
 
 - Fix issue with dispatch_loop and delays

data/bin/cli/sqs.rb

@@ -96,8 +96,8 @@ module Shoryuken
       end
 
       desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
-      method_option :watch,          aliases: '-w',  type: :boolean,              desc: 'watch queues'
-      method_option :watch_interval,                 type: :numeric, default: 10, desc: 'watch interval'
+      method_option :watch,    aliases: '-w', type: :boolean, desc: 'watch queues'
+      method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
       def ls(queue_name_prefix = '')
         trap('SIGINT', 'EXIT') # expect ctrl-c from loop
 
@@ -108,7 +108,7 @@ module Shoryuken
 
           break unless options[:watch]
 
-          sleep options[:watch_interval]
+          sleep options[:interval]
           puts
         end
       end
@@ -187,8 +187,12 @@ module Shoryuken
       end
 
       desc 'create QUEUE-NAME', 'Create a queue'
+      method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
       def create(queue_name)
-        queue_url = sqs.create_queue(queue_name: queue_name).queue_url
+        attributes = options[:attributes]
+        attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
+
+        queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
 
         say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
       end

data/bin/shoryuken

@@ -32,7 +32,7 @@ module Shoryuken
         opts = options.to_h.symbolize_keys
 
         if opts[:config_file]
-          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+          say '[DEPRECATED] Please use --config instead of --config-file', :yellow
         end
 
         opts[:config_file] = opts.delete(:config) if opts[:config]

data/lib/shoryuken/fetcher.rb

@@ -4,8 +4,6 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    attr_reader :group
-
     def initialize(group)
       @group = group
     end
@@ -26,12 +24,9 @@ module Shoryuken
     private
 
     def receive_messages(queue, limit)
-      # AWS limits the batch size by 10
-      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
-
-      options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
+      options = receive_options(queue)
 
-      options[:max_number_of_messages]  = limit
+      options[:max_number_of_messages]  = max_number_of_messages(limit, options)
       options[:message_attribute_names] = %w(All)
       options[:attribute_names]         = %w(All)
 
@@ -39,5 +34,16 @@ module Shoryuken
 
       Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
+
+    def max_number_of_messages(limit, options)
+      [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
+    end
+
+    def receive_options(queue)
+      options = Shoryuken.sqs_client_receive_message_opts[queue.name]
+      options ||= Shoryuken.sqs_client_receive_message_opts[@group]
+
+      options.to_h.dup
+    end
   end
 end

data/lib/shoryuken/options.rb

@@ -57,7 +57,18 @@ module Shoryuken
       end
 
       def polling_strategy(group)
-        options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
+        strategy = (group == 'default' ? options : options[:groups][group]).to_h[:polling_strategy]
+
+        case strategy
+        when 'WeightedRoundRobin', nil # Default case
+          Polling::WeightedRoundRobin
+        when 'StrictPriority'
+          Polling::StrictPriority
+        when Class
+          strategy
+        else
+          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+        end
       end
 
       def start_callback

data/lib/shoryuken/runner.rb

@@ -118,10 +118,12 @@ module Shoryuken
     end
 
     def handle_signal(sig)
+      logger.debug "Got #{sig} signal"
+
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      else
+      when 'TERM', 'INT'
         logger.info { "Received #{sig}, will shutdown down" }
 
         raise Interrupt

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.1.6'.freeze
+  VERSION = '3.1.7'.freeze
 end

data/spec/shoryuken/fetcher_spec.rb

@@ -2,6 +2,7 @@ require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
+# rubocop:disable Metrics/BlockLength
 RSpec.describe Shoryuken::Fetcher do
   let(:queue)        { instance_double('Shoryuken::Queue') }
   let(:queue_name)   { 'default' }
@@ -13,7 +14,7 @@ RSpec.describe Shoryuken::Fetcher do
       Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
-      message_id: 'fc754df79cc24c4196ca5996a44b771e',
+      message_id: 'fc754df79cc24c4196ca5996a44b771e'
     )
   end
 
@@ -27,23 +28,58 @@ RSpec.describe Shoryuken::Fetcher do
 
       Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
 
-      expect(queue).to receive(:receive_messages).
-        with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
-        and_return([])
+      expect(queue).to receive(:receive_messages).with(
+        wait_time_seconds: 10,
+        max_number_of_messages: limit,
+        message_attribute_names: ['All'],
+        attribute_names: ['All']
+      ).and_return([])
 
       subject.fetch(queue_config, limit)
     end
 
+    context 'when receive options per queue' do
+      let(:limit) { 5 }
+
+      specify do
+        expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+        Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 1 }
+
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: 1,
+          message_attribute_names: ['All'],
+          attribute_names: ['All']
+        ).and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
+    end
+
+    context 'when max_number_of_messages opt is great than limit' do
+      it 'uses limit' do
+        expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+        Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 20 }
+
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: limit,
+          message_attribute_names: ['All'],
+          attribute_names: ['All']
+        ).and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
+    end
+
     context 'when limit is greater than FETCH_LIMIT' do
       let(:limit) { 20 }
 
       specify do
-        Shoryuken.sqs_client_receive_message_opts[group] = {}
-
         allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-        expect(queue).to receive(:receive_messages).
-          with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
-          and_return([])
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']
+        ).and_return([])
 
         subject.fetch(queue_config, limit)
       end

data/spec/shoryuken/options_spec.rb

@@ -97,4 +97,84 @@ RSpec.describe Shoryuken::Options do
         "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
     end
   end
+
+  describe '.polling_strategy' do
+    context 'when not set' do
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
+      end
+    end
+
+    context 'when set to StrictPriority string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'StrictPriority'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'StrictPriority'
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::StrictPriority
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::StrictPriority
+      end
+    end
+
+    context 'when set to WeightedRoundRobin string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'WeightedRoundRobin'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'WeightedRoundRobin'
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
+      end
+    end
+
+    context 'when set to non-existent string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'NonExistent1'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'NonExistent2'
+          }
+        }
+      end
+
+      specify do
+        expect { Shoryuken.polling_strategy('default') }.to raise_error(ArgumentError)
+        expect { Shoryuken.polling_strategy('group1') }.to raise_error(ArgumentError)
+      end
+    end
+
+    context 'when set to a class' do
+      before do
+        class Foo < Shoryuken::Polling::BaseStrategy; end
+        class Bar < Shoryuken::Polling::BaseStrategy; end
+
+        Shoryuken.options[:polling_strategy] = Foo
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: Bar
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Foo
+        expect(Shoryuken.polling_strategy('group1')).to eq Bar
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -59,6 +59,8 @@ RSpec.configure do |config|
 
     Aws.config[:stub_responses] = true
 
+    Shoryuken.sqs_client_receive_message_opts.clear
+
     allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
     allow(Shoryuken).to receive(:active_job?).and_return(false)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 3.1.6
+  version: 3.1.7
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-24 00:00:00.000000000 Z
+date: 2017-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler