shoryuken 5.0.4 → 5.2.3

data/spec/shared_examples_for_active_job.rb

@@ -1,30 +1,44 @@
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+
+# Stand-in for a job class specified by the user
+class TestJob < ActiveJob::Base; end
+
 # rubocop:disable Metrics/BlockLength
 RSpec.shared_examples 'active_job_adapters' do
-  let(:job) { double 'Job', id: '123', queue_name: 'queue' }
+  let(:job_sqs_send_message_parameters) { {} }
+  let(:job) do
+    job = TestJob.new
+    job.sqs_send_message_parameters = job_sqs_send_message_parameters
+    job
+  end
   let(:fifo) { false }
   let(:queue) { double 'Queue', fifo?: fifo }
 
   before do
     allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
-    allow(job).to receive(:serialize).and_return(
-      'job_class' => 'Worker',
-      'job_id' => job.id,
-      'queue_name' => job.queue_name,
-      'arguments' => nil,
-      'locale' => nil
-    )
   end
 
   describe '#enqueue' do
     specify do
       expect(queue).to receive(:send_message) do |hash|
         expect(hash[:message_deduplication_id]).to_not be
+        expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
+        expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
+        expect(hash[:message_attributes].keys).to eq(['shoryuken_class'])
       end
       expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
 
       subject.enqueue(job)
     end
 
+    it "should mutate the job's sqs_send_message_parameters reference to match those sent to the queue" do
+      expect(queue).to receive(:send_message) do |options|
+        expect(options).to be(job.sqs_send_message_parameters)
+      end
+      subject.enqueue(job)
+    end
+
     context 'when fifo' do
       let(:fifo) { true }
 
@@ -38,6 +52,209 @@ RSpec.shared_examples 'active_job_adapters' do
 
         subject.enqueue(job)
       end
+
+      context 'with message_deduplication_id' do
+        context 'when message_deduplication_id is specified in options' do
+          it 'should enqueue a message with the deduplication_id specified in options' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
+            end
+            subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
+          end
+        end
+
+        context 'when message_deduplication_id is specified on the job' do
+          let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
+
+          it 'should enqueue a message with the deduplication_id specified on the job' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('job-dedupe-id')
+            end
+            subject.enqueue job
+          end
+        end
+
+        context 'when message_deduplication_id is specified on the job and also in options' do
+          let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
+
+          it 'should enqueue a message with the deduplication_id specified in options' do
+            expect(queue).to receive(:send_message) do |hash|
+              expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
+            end
+            subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
+          end
+        end
+      end
+    end
+
+    context 'with message_group_id' do
+      context 'when message_group_id is specified in options' do
+        it 'should enqueue a message with the group_id specified in options' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('options-group-id')
+          end
+          subject.enqueue(job, message_group_id: 'options-group-id')
+        end
+      end
+
+      context 'when message_group_id is specified on the job' do
+        let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
+
+        it 'should enqueue a message with the group_id specified on the job' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('job-group-id')
+          end
+          subject.enqueue job
+        end
+      end
+
+      context 'when message_group_id is specified on the job and also in options' do
+        let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
+
+        it 'should enqueue a message with the group_id specified in options' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_group_id]).to eq('options-group-id')
+          end
+          subject.enqueue(job, message_group_id: 'options-group-id')
+        end
+      end
+    end
+
+    context 'with additional message attributes' do
+      it 'should combine with activejob attributes' do
+        custom_message_attributes = {
+          'tracer_id' => {
+            string_value: SecureRandom.hex,
+            data_type: 'String'
+          }
+        }
+
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
+          expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
+          expect(hash[:message_attributes]['tracer_id'][:string_value]).to eq(custom_message_attributes['tracer_id'][:string_value])
+          expect(hash[:message_attributes]['tracer_id'][:data_type]).to eq("String")
+        end
+        expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+        subject.enqueue(job, message_attributes: custom_message_attributes)
+      end
+
+      context 'when message_attributes are specified on the job' do
+        let(:job_sqs_send_message_parameters) do
+          {
+            message_attributes: {
+              'tracer_id' => {
+                data_type: 'String',
+                string_value: 'job-value'
+              }
+            }
+          }
+        end
+
+        it 'should enqueue a message with the message_attributes specified on the job' do
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_attributes]['tracer_id']).to eq({ data_type: 'String', string_value: 'job-value' })
+            expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
+          end
+          subject.enqueue job
+        end
+      end
+
+      context 'when message_attributes are specified on the job and also in options' do
+        let(:job_sqs_send_message_parameters) do
+          {
+            message_attributes: {
+              'tracer_id' => {
+                data_type: 'String',
+                string_value: 'job-value'
+              }
+            }
+          }
+        end
+
+        it 'should enqueue a message with the message_attributes speficied in options' do
+          custom_message_attributes = {
+            'options_tracer_id' => {
+              string_value: 'options-value',
+              data_type: 'String'
+            }
+          }
+
+          expect(queue).to receive(:send_message) do |hash|
+            expect(hash[:message_attributes]['tracer_id']).to be_nil
+            expect(hash[:message_attributes]['options_tracer_id']).to eq({ data_type: 'String', string_value: 'options-value' })
+            expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
+          end
+          subject.enqueue job, message_attributes: custom_message_attributes
+        end
+      end
+    end
+  end
+
+  context 'with message_system_attributes' do
+    context 'when message_system_attributes are specified in options' do
+      it 'should enqueue a message with message_system_attributes specified in options' do
+        system_attributes = {
+          'AWSTraceHeader' => {
+            string_value: 'trace_id',
+            data_type: 'String'
+          }
+        }
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['AWSTraceHeader'][:string_value]).to eq('trace_id')
+          expect(hash[:message_system_attributes]['AWSTraceHeader'][:data_type]).to eq('String')
+        end
+        subject.enqueue(job, message_system_attributes: system_attributes)
+      end
+    end
+
+    context 'when message_system_attributes are specified on the job' do
+      let(:job_sqs_send_message_parameters) do
+        {
+          message_system_attributes: {
+            'AWSTraceHeader' => {
+              string_value: 'job-value',
+              data_type: 'String'
+            }
+          }
+        }
+      end
+
+      it 'should enqueue a message with the message_system_attributes specified on the job' do
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['AWSTraceHeader']).to eq({ data_type: 'String', string_value: 'job-value' })
+        end
+        subject.enqueue job
+      end
+    end
+
+    context 'when message_system_attributes are specified on the job and also in options' do
+      let(:job_sqs_send_message_parameters) do
+        {
+          message_system_attributes: {
+            'job_trace_header' => {
+              string_value: 'job-value',
+              data_type: 'String'
+            }
+          }
+        }
+      end
+
+      it 'should enqueue a message with the message_system_attributes speficied in options' do
+        custom_message_attributes = {
+          'options_trace_header' => {
+            string_value: 'options-value',
+            data_type: 'String'
+          }
+        }
+
+        expect(queue).to receive(:send_message) do |hash|
+          expect(hash[:message_system_attributes]['job_trace_header']).to be_nil
+          expect(hash[:message_system_attributes]['options_trace_header']).to eq({ data_type: 'String', string_value: 'options-value' })
+        end
+        subject.enqueue job, message_system_attributes: custom_message_attributes
+      end
     end
   end
 
@@ -59,4 +276,4 @@ RSpec.shared_examples 'active_job_adapters' do
     end
   end
 end
-# rubocop:enable Metrics/BlockLength
+# rubocop:enable Metrics/BlockLength

data/spec/shoryuken/environment_loader_spec.rb

@@ -57,11 +57,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       ActiveJob::Base.queue_name_delimiter = '_'
 
       allow(Shoryuken).to receive(:active_job?).and_return(true)
-    end
 
-    specify do
       Shoryuken.active_job_queue_name_prefixing = true
+    end
 
+    specify do
       Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
 
       Shoryuken.options[:groups] = {
@@ -73,6 +73,26 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
       expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
     end
+
+    it 'does not prefix url-based queues' do
+      Shoryuken.options[:queues] = ['https://example.com/test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
+    end
+
+    it 'does not prefix arn-based queues' do
+      Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
+    end
   end
   describe "#setup_options" do
     let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }

data/spec/shoryuken/extensions/active_job_adapter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
-require 'shoryuken/extensions/active_job_adapter'
 require 'shared_examples_for_active_job'
+require 'shoryuken/extensions/active_job_adapter'
 
 RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
   include_examples 'active_job_adapters'

data/spec/shoryuken/extensions/active_job_base_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+require 'shoryuken/extensions/active_job_adapter'
+
+RSpec.describe ActiveJob::Base do
+  let(:queue_adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new }
+
+  subject do
+    worker_class = Class.new(described_class)
+    Object.const_set :MyWorker, worker_class
+    worker_class.queue_adapter = queue_adapter
+    worker_class
+  end
+
+  after do
+    Object.send :remove_const, :MyWorker
+  end
+
+  describe '#perform_now' do
+    it 'allows keyward args' do
+      collaborator = double 'worker collaborator'
+      subject.send(:define_method, :perform) do |**kwargs|
+        collaborator.foo(**kwargs)
+      end
+      expect(collaborator).to receive(:foo).with(foo: 'bar')
+      subject.perform_now foo: 'bar'
+    end
+  end
+
+  describe '#perform_later' do
+    it 'calls enqueue on the adapter with the expected job' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.arguments).to eq([1, 2])
+      end
+
+      subject.perform_later 1, 2
+    end
+
+    it 'passes message_group_id to the queue_adapter' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_group_id]).to eq('group-2')
+      end
+
+      subject.set(message_group_id: 'group-2').perform_later 1, 2
+    end
+
+    it 'passes message_deduplication_id to the queue_adapter' do
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_deduplication_id]).to eq('dedupe-id')
+      end
+
+      subject.set(message_deduplication_id: 'dedupe-id').perform_later 1, 2
+    end
+
+    it 'passes message_attributes to the queue_adapter' do
+      message_attributes = {
+        'custom_tracing_id' => {
+          string_value: 'value',
+          data_type: 'String'
+        }
+      }
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_attributes]).to eq(message_attributes)
+      end
+
+      subject.set(message_attributes: message_attributes).perform_later 1, 2
+    end
+
+    it 'passes message_system_attributes to the queue_adapter' do
+      message_system_attributes = {
+        'AWSTraceHeader' => {
+          string_value: 'trace_id',
+          data_type: 'String'
+        }
+      }
+      expect(queue_adapter).to receive(:enqueue) do |job|
+        expect(job.sqs_send_message_parameters[:message_system_attributes]).to eq(message_system_attributes)
+      end
+
+      subject.set(message_system_attributes: message_system_attributes).perform_later 1, 2
+    end
+  end
+end

data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb

@@ -10,6 +10,10 @@ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
   let(:error_handler) { -> {} }
   let(:success_handler) { -> {} }
 
+  before do
+    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
+  end
+
   subject { described_class.new(success_handler, error_handler) }
 
   context 'when success' do

data/spec/shoryuken/extensions/active_job_wrapper_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+require 'shoryuken/extensions/active_job_adapter'
+
+RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper do
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'sets executions to reflect approximate receive count' do
+      attributes = { 'ApproximateReceiveCount' => '42' }
+      sqs_msg = double Shoryuken::Message, attributes: attributes
+      job_hash = { 'arguments' => [1, 2, 3] }
+      job_hash_with_executions = { 'arguments' => [1, 2, 3], 'executions' => 41 }
+      expect(ActiveJob::Base).to receive(:execute).with(job_hash_with_executions)
+
+      subject.perform sqs_msg, job_hash
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
   let(:concurrency) { 1 }
   let(:executor) { Concurrent::ImmediateExecutor.new }
 
-  subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
+  subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
 
   before do
     allow(fetcher).to receive(:fetch).and_return([])
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
 
       expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+      expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                   false,
+                                                   group: 'default',
+                                                   busy_processors: 1,
+                                                   max_processors: 1)
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
       expect(Shoryuken.logger).to receive(:info).never
 
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
         q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
 
         expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+        expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                     false,
+                                                     group: 'default',
+                                                     busy_processors: 1,
+                                                     max_processors: 1)
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)
@@ -139,4 +149,28 @@ RSpec.describe Shoryuken::Manager do
       subject.send(:dispatch_single_messages, q)
     end
   end
+
+  describe '#processor_done' do
+    let(:sqs_queue)         { double Shoryuken::Queue }
+
+    before do
+      allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+    end
+
+    context 'when queue.fifo? is true' do
+      it 'calls message_processed on strategy' do
+        expect(sqs_queue).to receive(:fifo?).and_return(true)
+        expect(polling_strategy).to receive(:message_processed).with(queue)
+        subject.send(:processor_done, queue)
+      end
+    end
+
+    context 'when queue.fifo? is false' do
+      it 'does not call message_processed on strategy' do
+        expect(sqs_queue).to receive(:fifo?).and_return(false)
+        expect(polling_strategy).to_not receive(:message_processed)
+        subject.send(:processor_done, queue)
+      end
+    end
+  end
 end