shoryuken 5.0.1 → 5.1.0

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -35,10 +35,20 @@ module Shoryuken
         unparse_queues(@queues)
       end
 
+      def message_processed(queue)
+        return if @paused_queues.empty?
+
+        logger.debug "Unpausing #{queue}"
+        @paused_queues.reject! { |_time, name| name == queue }
+        @queues << queue
+        @queues.uniq!
+      end
+
       private
 
       def pause(queue)
         return unless @queues.delete(queue)
+
         @paused_queues << [Time.now + delay, queue]
         logger.debug "Paused #{queue}"
       end
@@ -46,6 +56,7 @@ module Shoryuken
       def unpause_queues
         return if @paused_queues.empty?
         return if Time.now < @paused_queues.first[0]
+
         pause = @paused_queues.shift
         @queues << pause[1]
         logger.debug "Unpaused #{pause[1]}"

data/lib/shoryuken/queue.rb

@@ -8,9 +8,9 @@ module Shoryuken
 
     attr_accessor :name, :client, :url
 
-    def initialize(client, name_or_url)
+    def initialize(client, name_or_url_or_arn)
       self.client = client
-      set_name_and_url(name_or_url)
+      set_name_and_url(name_or_url_or_arn)
     end
 
     def visibility_timeout
@@ -50,32 +50,60 @@ module Shoryuken
       # Make sure the memoization work with boolean to avoid multiple calls to SQS
       # see https://github.com/phstc/shoryuken/pull/529
       return @_fifo if defined?(@_fifo)
+
       @_fifo = queue_attributes.attributes[FIFO_ATTR] == 'true'
+      @_fifo
     end
 
     private
 
-    def set_by_name(name)
+    def initialize_fifo_attribute
+      # calling fifo? will also initialize it
+      fifo?
+    end
+
+    def set_by_name(name) # rubocop:disable Naming/AccessorMethodName
       self.name = name
       self.url  = client.get_queue_url(queue_name: name).queue_url
     end
 
-    def set_by_url(url)
+    def set_by_url(url) # rubocop:disable Naming/AccessorMethodName
       self.name = url.split('/').last
       self.url  = url
     end
 
-    def set_name_and_url(name_or_url)
-      if name_or_url.include?('://')
-        set_by_url(name_or_url)
+    def arn_to_url(arn_str)
+      *, region, account_id, resource = arn_str.split(':')
+
+      required = [region, account_id, resource].map(&:to_s)
+      valid = required.none?(&:empty?)
+
+      abort "Invalid ARN: #{arn_str}. A valid ARN must include: region, account_id and resource." unless valid
+
+      "https://sqs.#{region}.amazonaws.com/#{account_id}/#{resource}"
+    end
+
+    def set_name_and_url(name_or_url_or_arn) # rubocop:disable Naming/AccessorMethodName
+      if name_or_url_or_arn.include?('://')
+        set_by_url(name_or_url_or_arn)
+
+        # anticipate the fifo? checker for validating the queue URL
+        initialize_fifo_attribute
+        return
+      end
+
+      if name_or_url_or_arn.start_with?('arn:')
+        url = arn_to_url(name_or_url_or_arn)
+        set_by_url(url)
 
         # anticipate the fifo? checker for validating the queue URL
-        return fifo?
+        initialize_fifo_attribute
+        return
       end
 
-      set_by_name(name_or_url)
-    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
-      raise ex, "The specified queue #{name_or_url} does not exist."
+      set_by_name(name_or_url_or_arn)
+    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => e
+      raise e, "The specified queue #{name_or_url_or_arn} does not exist."
     end
 
     def queue_attributes

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.0.1'.freeze
+  VERSION = '5.1.0'.freeze
 end

data/lib/shoryuken.rb

@@ -88,4 +88,8 @@ module Shoryuken
   )
 end
 
-require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?
+if Shoryuken.active_job?
+  require 'shoryuken/extensions/active_job_extensions'
+  require 'shoryuken/extensions/active_job_adapter'
+  require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+end

data/shoryuken.gemspec

@@ -18,7 +18,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'dotenv'
-  spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
 

data/spec/shared_examples_for_active_job.rb

@@ -0,0 +1,279 @@
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+
+# Stand-in for a job class specified by the user
+class TestJob < ActiveJob::Base; end
+
+# rubocop:disable Metrics/BlockLength
+RSpec.shared_examples 'active_job_adapters' do
+  let(:job_sqs_send_message_parameters) { {} }
+  let(:job) do
+    job = TestJob.new
+    job.sqs_send_message_parameters = job_sqs_send_message_parameters
+    job
+  end
+  let(:fifo) { false }
+  let(:queue) { double 'Queue', fifo?: fifo }
+
+  before do
+    allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
+  end
+
+  describe '#enqueue' do
+    specify do
+      expect(queue).to receive(:send_message) do |hash|
+        expect(hash[:message_deduplication_id]).to_not be
+        expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
+        expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
+        expect(hash[:message_attributes].keys).to eq(['shoryuken_class'])
+      end
+      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+      subject.enqueue(job)
+    end
+
+    it "should mutate the job's sqs_send_message_parameters reference to match those sent to the queue" do
+      expect(queue).to receive(:send_message) do |options|
+        expect(options).to be(job.sqs_send_message_parameters)
+      end
+      subject.enqueue(job)
+    end
+
+    context 'when fifo' do
+      let(:fifo) { true }
+
+      it 'does not include job_id in the deduplication_id' do
+        expect(queue).to receive(:send_message) do |hash|
+          message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
+
+          expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
+        end
+        expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+        subject.enqueue(job)
+      end
+
+      context 'with message_deduplication_id' do
+        context 'when message_deduplication_id is specified in options' do
+          it 'should enqueue a message with the deduplication_id specified in options' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
+            end
+            subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
+          end
+        end
+
+        context 'when message_deduplication_id is specified on the job' do
+          let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
+
+          it 'should enqueue a message with the deduplication_id specified on the job' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('job-dedupe-id')
+            end
+            subject.enqueue job
+          end
+        end
+
+        context 'when message_deduplication_id is specified on the job and also in options' do
+          let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
+
+          it 'should enqueue a message with the deduplication_id specified in options' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
+            end
+            subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
+          end
+        end
+      end
+    end
+
+    context 'with message_group_id' do
+      context 'when message_group_id is specified in options' do
+        it 'should enqueue a message with the group_id specified in options' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('options-group-id')
+          end
+          subject.enqueue(job, message_group_id: 'options-group-id')
+        end
+      end
+
+      context 'when message_group_id is specified on the job' do
+        let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
+
+        it 'should enqueue a message with the group_id specified on the job' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('job-group-id')
+          end
+          subject.enqueue job
+        end
+      end
+
+      context 'when message_group_id is specified on the job and also in options' do
+        let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
+
+        it 'should enqueue a message with the group_id specified in options' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('options-group-id')
+          end
+          subject.enqueue(job, message_group_id: 'options-group-id')
+        end
+      end
+    end
+
+    context 'with additional message attributes' do
+      it 'should combine with activejob attributes' do
+        custom_message_attributes = {
+          'tracer_id' => {
+            string_value: SecureRandom.hex,
+            data_type: 'String'
+          }
+        }
+
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
+          expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
+          expect(hash[:message_attributes]['tracer_id'][:string_value]).to eq(custom_message_attributes['tracer_id'][:string_value])
+          expect(hash[:message_attributes]['tracer_id'][:data_type]).to eq("String")
+        end
+        expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+        subject.enqueue(job, message_attributes: custom_message_attributes)
+      end
+
+      context 'when message_attributes are specified on the job' do
+        let(:job_sqs_send_message_parameters) do
+          {
+            message_attributes: {
+              'tracer_id' => {
+                data_type: 'String',
+                string_value: 'job-value'
+              }
+            }
+          }
+        end
+
+        it 'should enqueue a message with the message_attributes specified on the job' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_attributes]['tracer_id']).to eq({ data_type: 'String', string_value: 'job-value' })
+            expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
+          end
+          subject.enqueue job
+        end
+      end
+
+      context 'when message_attributes are specified on the job and also in options' do
+        let(:job_sqs_send_message_parameters) do
+          {
+            message_attributes: {
+              'tracer_id' => {
+                data_type: 'String',
+                string_value: 'job-value'
+              }
+            }
+          }
+        end
+
+        it 'should enqueue a message with the message_attributes speficied in options' do
+          custom_message_attributes = {
+            'options_tracer_id' => {
+              string_value: 'options-value',
+              data_type: 'String'
+            }
+          }
+
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_attributes]['tracer_id']).to be_nil
+            expect(hash[:message_attributes]['options_tracer_id']).to eq({ data_type: 'String', string_value: 'options-value' })
+            expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
+          end
+          subject.enqueue job, message_attributes: custom_message_attributes
+        end
+      end
+    end
+  end
+
+  context 'with message_system_attributes' do
+    context 'when message_system_attributes are specified in options' do
+      it 'should enqueue a message with message_system_attributes specified in options' do
+        system_attributes = {
+          'AWSTraceHeader' => {
+            string_value: 'trace_id',
+            data_type: 'String'
+          }
+        }
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['AWSTraceHeader'][:string_value]).to eq('trace_id')
+          expect(hash[:message_system_attributes]['AWSTraceHeader'][:data_type]).to eq('String')
+        end
+        subject.enqueue(job, message_system_attributes: system_attributes)
+      end
+    end
+
+    context 'when message_system_attributes are specified on the job' do
+      let(:job_sqs_send_message_parameters) do
+        {
+          message_system_attributes: {
+            'AWSTraceHeader' => {
+              string_value: 'job-value',
+              data_type: 'String'
+            }
+          }
+        }
+      end
+
+      it 'should enqueue a message with the message_system_attributes specified on the job' do
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['AWSTraceHeader']).to eq({ data_type: 'String', string_value: 'job-value' })
+        end
+        subject.enqueue job
+      end
+    end
+
+    context 'when message_system_attributes are specified on the job and also in options' do
+      let(:job_sqs_send_message_parameters) do
+        {
+          message_system_attributes: {
+            'job_trace_header' => {
+              string_value: 'job-value',
+              data_type: 'String'
+            }
+          }
+        }
+      end
+
+      it 'should enqueue a message with the message_system_attributes speficied in options' do
+        custom_message_attributes = {
+          'options_trace_header' => {
+            string_value: 'options-value',
+            data_type: 'String'
+          }
+        }
+
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['job_trace_header']).to be_nil
+          expect(hash[:message_system_attributes]['options_trace_header']).to eq({ data_type: 'String', string_value: 'options-value' })
+        end
+        subject.enqueue job, message_system_attributes: custom_message_attributes
+      end
+    end
+  end
+
+  describe '#enqueue_at' do
+    specify do
+      delay = 1
+
+      expect(queue).to receive(:send_message) do |hash|
+        expect(hash[:message_deduplication_id]).to_not be
+        expect(hash[:delay_seconds]).to eq(delay)
+      end
+
+      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+      # need to figure out what to require Time.current and N.minutes to remove the stub
+      allow(subject).to receive(:calculate_delay).and_return(delay)
+
+      subject.enqueue_at(job, nil)
+    end
+  end
+end
+# rubocop:enable Metrics/BlockLength

data/spec/shoryuken/environment_loader_spec.rb

@@ -74,4 +74,28 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
     end
   end
+  describe "#setup_options" do
+    let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }
+    let (:config_queues) { [["queue1", 8], ["queue2", 4]] }
+    context "when given queues through config and CLI" do
+      specify do
+        allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
+        Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
+        expect(Shoryuken.options[:queues]).to eq(cli_queues)
+      end
+    end
+    context "when given queues through config only" do
+      specify do
+        allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
+        Shoryuken::EnvironmentLoader.setup_options({})
+        expect(Shoryuken.options[:queues]).to eq(config_queues)
+      end
+    end
+    context "when given queues through CLI only" do
+      specify do
+        Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
+        expect(Shoryuken.options[:queues]).to eq(cli_queues)
+      end
+    end
+  end
 end

data/spec/shoryuken/extensions/active_job_adapter_spec.rb

@@ -1,64 +1,7 @@
 require 'spec_helper'
-require 'active_job'
+require 'shared_examples_for_active_job'
 require 'shoryuken/extensions/active_job_adapter'
 
 RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
-  let(:job) { double 'Job', id: '123', queue_name: 'queue' }
-  let(:fifo) { false }
-  let(:queue) { double 'Queue', fifo?: fifo }
-
-  before do
-    allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
-    allow(job).to receive(:serialize).and_return(
-      'job_class' => 'Worker',
-      'job_id'     => job.id,
-      'queue_name' => job.queue_name,
-      'arguments'  => nil,
-      'locale'     => nil
-    )
-  end
-
-  describe '#enqueue' do
-    specify do
-      expect(queue).to receive(:send_message) do |hash|
-        expect(hash[:message_deduplication_id]).to_not be
-      end
-      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
-
-      subject.enqueue(job)
-    end
-
-    context 'when fifo' do
-      let(:fifo) { true }
-
-      it 'does not include job_id in the deduplication_id' do
-        expect(queue).to receive(:send_message) do |hash|
-          message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
-
-          expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
-        end
-        expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
-
-        subject.enqueue(job)
-      end
-    end
-  end
-
-  describe '#enqueue_at' do
-    specify do
-      delay = 1
-
-      expect(queue).to receive(:send_message) do |hash|
-        expect(hash[:message_deduplication_id]).to_not be
-        expect(hash[:delay_seconds]).to eq(delay)
-      end
-
-      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
-
-      # need to figure out what to require Time.current and N.minutes to remove the stub
-      allow(subject).to receive(:calculate_delay).and_return(delay)
-
-      subject.enqueue_at(job, nil)
-    end
-  end
+  include_examples 'active_job_adapters'
 end

data/spec/shoryuken/extensions/active_job_base_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+require 'shoryuken/extensions/active_job_adapter'
+
+RSpec.describe ActiveJob::Base do
+  let(:queue_adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new }
+
+  subject do
+    worker_class = Class.new(described_class)
+    Object.const_set :MyWorker, worker_class
+    worker_class.queue_adapter = queue_adapter
+    worker_class
+  end
+
+  after do
+    Object.send :remove_const, :MyWorker
+  end
+
+  describe '#perform_later' do
+    it 'calls enqueue on the adapter with the expected job' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.arguments).to eq([1, 2])
+      end
+
+      subject.perform_later 1, 2
+    end
+
+    it 'passes message_group_id to the queue_adapter' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_group_id]).to eq('group-2')
+      end
+
+      subject.set(message_group_id: 'group-2').perform_later 1, 2
+    end
+
+    it 'passes message_deduplication_id to the queue_adapter' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_deduplication_id]).to eq('dedupe-id')
+      end
+
+      subject.set(message_deduplication_id: 'dedupe-id').perform_later 1, 2
+    end
+
+    it 'passes message_attributes to the queue_adapter' do
+      message_attributes = {
+        'custom_tracing_id' => {
+          string_value: 'value',
+          data_type: 'String'
+        }
+      }
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_attributes]).to eq(message_attributes)
+      end
+
+      subject.set(message_attributes: message_attributes).perform_later 1, 2
+    end
+
+    it 'passes message_system_attributes to the queue_adapter' do
+      message_system_attributes = {
+        'AWSTraceHeader' => {
+          string_value: 'trace_id',
+          data_type: 'String'
+        }
+      }
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_system_attributes]).to eq(message_system_attributes)
+      end
+
+      subject.set(message_system_attributes: message_system_attributes).perform_later 1, 2
+    end
+  end
+end

data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'shared_examples_for_active_job'
+require 'shoryuken/extensions/active_job_adapter'
+require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+
+RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
+  include_examples 'active_job_adapters'
+
+  let(:options) { {} }
+  let(:error_handler) { -> {} }
+  let(:success_handler) { -> {} }
+
+  subject { described_class.new(success_handler, error_handler) }
+
+  context 'when success' do
+    it 'calls success_handler' do
+      response = true
+      allow(queue).to receive(:send_message).and_return(response)
+      expect(success_handler).to receive(:call).with(response, job, options)
+
+      subject.enqueue(job, options)
+    end
+  end
+
+  context 'when failure' do
+    it 'calls error_handler' do
+      response = Aws::SQS::Errors::InternalError.new('error', 'error')
+
+      allow(queue).to receive(:send_message).and_raise(response)
+      expect(error_handler).to receive(:call).with(response, job, options).and_call_original
+
+      subject.enqueue(job, options)
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -139,4 +139,28 @@ RSpec.describe Shoryuken::Manager do
       subject.send(:dispatch_single_messages, q)
     end
   end
+
+  describe '#processor_done' do
+    let(:sqs_queue)         { double Shoryuken::Queue }
+
+    before do
+      allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+    end
+
+    context 'when queue.fifo? is true' do
+      it 'calls message_processed on strategy' do
+        expect(sqs_queue).to receive(:fifo?).and_return(true)
+        expect(polling_strategy).to receive(:message_processed).with(queue)
+        subject.send(:processor_done, queue)
+      end
+    end
+
+    context 'when queue.fifo? is false' do
+      it 'does not call message_processed on strategy' do
+        expect(sqs_queue).to receive(:fifo?).and_return(false)
+        expect(polling_strategy).to_not receive(:message_processed)
+        subject.send(:processor_done, queue)
+      end
+    end
+  end
 end

data/spec/shoryuken/polling/strict_priority_spec.rb

@@ -145,4 +145,14 @@ RSpec.describe Shoryuken::Polling::StrictPriority do
       expect(subject.next_queue).to eq(queue3)
     end
   end
+
+  describe '#message_processed' do
+    it 'removes paused queue, adds to active queues' do
+      strategy = Shoryuken::Polling::StrictPriority.new([queue1, queue2])
+      strategy.send(:pause, queue1)
+      expect(strategy.active_queues).to eq([[queue2, 1]])
+      strategy.message_processed(queue1)
+      expect(strategy.active_queues).to eq([[queue1, 2], [queue2, 1]])
+    end
+  end
 end

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -104,4 +104,14 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
       expect(subject.delay).to eq(1.0)
     end
   end
+
+  describe '#message_processed' do
+    it 'removes paused queue, adds to active queues' do
+      strategy = Shoryuken::Polling::WeightedRoundRobin.new([queue1, queue2])
+      strategy.send(:pause, queue1)
+      expect(strategy.active_queues).to eq([[queue2, 1]])
+      strategy.message_processed(queue1)
+      expect(strategy.active_queues).to eq([[queue2, 1], [queue1, 1]])
+    end
+  end
 end