shoryuken 3.2.0 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
-  data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
+  metadata.gz: 853cbc2248381c89a862aa63acfdeb632703d816
+  data.tar.gz: b1001217f9763abf0e64049842331c6b856e08af
 SHA512:
-  metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
-  data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31
+  metadata.gz: 92b37bae40181a8a5225618282904de81591e8c7a02503cad00f3c1c3c24dc33b428482ae03d6ba59aacf1e1f14142205313f85f9378becd7965a3f4caaeb698
+  data.tar.gz: 8d1fb407c004db0b992b2437043395673802fb5bb6d8065d691025469c05ea43bf517f96f7ae46bbd5fbdb1fc9ec74abe2788e38dccfdcacf55b88ff632b887d

data/.travis.yml

@@ -3,9 +3,10 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1.10
-  - 2.2.0
-  - 2.3.3
-  - 2.4.1
+  - 2.2.9
+  - 2.3.6
+  - 2.4.3
+  - 2.5.0
 
 notifications:
   email:

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [v3.2.1] - 2018-02-12
+
+- Support FIFO queues in `shoryuken sqs` commands
+ - [#473](https://github.com/phstc/shoryuken/pull/473)
+
+- Allow customizing the default executor launcher
+ - [#469](https://github.com/phstc/shoryuken/pull/469)
+
+- Exclude job_id from message deduplication when ActiveJob
+ - [#462](https://github.com/phstc/shoryuken/pull/462)
+
 ## [v3.2.0] - 2018-01-03
 
 - Preserve parent worker class options

data/README.md

@@ -31,6 +31,14 @@ Add this line to your application's Gemfile:
 gem 'shoryuken'
 ```
 
+If you are using AWS SDK version 3, please also add this line:
+
+```ruby
+gem 'aws-sdk-sqs'
+```
+
+The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3. 
+
 And then execute:
 
 ```shell

data/bin/cli/sqs.rb

@@ -8,12 +8,14 @@ module Shoryuken
 
       no_commands do
         def normalize_dump_message(message)
-          message[:id] = message.delete(:message_id)
-          message[:message_body] = message.delete(:body)
-          message.delete(:receipt_handle)
-          message.delete(:md5_of_body)
-          message.delete(:md5_of_message_attributes)
-          message
+          attributes = message[:attributes]
+          {
+            id: message[:message_id],
+            message_body: message[:body],
+            message_attributes: message[:message_attributes],
+            message_deduplication_id: attributes['MessageDeduplicationId'],
+            message_group_id: attributes['MessageGroupId']
+          }
         end
 
         def sqs
@@ -59,6 +61,7 @@ module Shoryuken
             messages = sqs.receive_message(
               queue_url: url,
               max_number_of_messages: batch_size,
+              attribute_names: ['All'],
               message_attribute_names: ['All']
             ).messages
 

data/lib/shoryuken.rb

@@ -53,6 +53,8 @@ module Shoryuken
     :worker_registry=,
     :worker_executor,
     :worker_executor=,
+    :launcher_executor,
+    :launcher_executor=,
     :polling_strategy,
     :start_callback,
     :start_callback=,

data/lib/shoryuken/client.rb

@@ -8,11 +8,15 @@ module Shoryuken
       end
 
       def sqs
-        @@sqs ||= Shoryuken.sqs_client
+        Shoryuken.sqs_client
       end
 
       def sqs=(sqs)
-        @@sqs = sqs
+        # Since the @@queues values (Shoryuken::Queue objects) are built referencing @@sqs, if it changes, we need to
+        #   re-build them on subsequent calls to `.queues(name)`.
+        @@queues = {}
+
+        Shoryuken.sqs_client = sqs
       end
     end
   end

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -30,30 +30,40 @@ module ActiveJob
         end
       end
 
-      def enqueue(job) #:nodoc:
+      def enqueue(job, options = {}) #:nodoc:
         register_worker!(job)
 
         queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(job))
+        queue.send_message(message(queue, job, options))
       end
 
       def enqueue_at(job, timestamp) #:nodoc:
-        register_worker!(job)
+        enqueue(job, delay_seconds: calculate_delay(timestamp))
+      end
+
+      private
 
+      def calculate_delay(timestamp)
         delay = (timestamp - Time.current.to_f).round
         raise 'The maximum allowed delay is 15 minutes' if delay > 15.minutes
 
-        queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(job, delay_seconds: delay))
+        delay
       end
 
-      private
-
-      def message(job, options = {})
+      def message(queue, job, options = {})
         body = job.serialize
 
-        { message_body: body,
-          message_attributes: message_attributes }.merge(options)
+        msg = {}
+
+        if queue.fifo?
+          # See https://github.com/phstc/shoryuken/issues/457
+          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
+        end
+
+        msg[:message_body] = body
+        msg[:message_attributes] = message_attributes
+
+        msg.merge(options)
       end
 
       def register_worker!(job)

data/lib/shoryuken/launcher.rb

@@ -35,7 +35,7 @@ module Shoryuken
     private
 
     def executor
-      Concurrent.global_io_executor
+      @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
 
     def start_managers

data/lib/shoryuken/options.rb

@@ -22,6 +22,7 @@ module Shoryuken
     @@start_callback                  = nil
     @@stop_callback                   = nil
     @@worker_executor                 = Worker::DefaultExecutor
+    @@launcher_executor               = nil
 
     class << self
       def active_job?
@@ -65,6 +66,14 @@ module Shoryuken
         @@worker_executor = worker_executor
       end
 
+      def launcher_executor
+        @@launcher_executor
+      end
+
+      def launcher_executor=(launcher_executor)
+        @@launcher_executor = launcher_executor
+      end
+
       def polling_strategy(group)
         strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.2.0'.freeze
+  VERSION = '3.2.1'.freeze
 end

data/spec/shoryuken/extensions/active_job_adapter_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+require 'active_job'
+require 'shoryuken/extensions/active_job_adapter'
+
+RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
+  let(:job) { double 'Job', id: '123', queue_name: 'queue' }
+  let(:fifo) { false }
+  let(:queue) { double 'Queue', fifo?: fifo }
+
+  before do
+    allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
+    allow(job).to receive(:serialize).and_return({
+      'job_class'  => 'Worker',
+      'job_id'     => job.id,
+      'queue_name' => job.queue_name,
+      'arguments'  => nil,
+      'locale'     => nil
+    })
+  end
+
+  describe '#enqueue' do
+    specify do
+      expect(queue).to receive(:send_message) do |hash|
+        expect(hash[:message_deduplication_id]).to_not be
+      end
+      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+      subject.enqueue(job)
+    end
+
+    context 'when fifo' do
+      let(:fifo) { true }
+
+      it 'does not include job_id in the deduplication_id' do
+        expect(queue).to receive(:send_message) do |hash|
+          message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
+
+          expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
+        end
+        expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+        subject.enqueue(job)
+      end
+    end
+  end
+
+  describe '#enqueue_at' do
+    specify do
+      delay = 1
+
+      expect(queue).to receive(:send_message) do |hash|
+        expect(hash[:message_deduplication_id]).to_not be
+        expect(hash[:delay_seconds]).to eq(delay)
+      end
+
+      expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
+
+      # need to figure out what to require Time.current and N.minutes to remove the stub
+      allow(subject).to receive(:calculate_delay).and_return(delay)
+
+      subject.enqueue_at(job, nil)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.2.1
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-03 00:00:00.000000000 Z
+date: 2018-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -186,6 +186,7 @@ files:
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
 - spec/shoryuken/environment_loader_spec.rb
+- spec/shoryuken/extensions/active_job_adapter_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
@@ -239,6 +240,7 @@ test_files:
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
 - spec/shoryuken/environment_loader_spec.rb
+- spec/shoryuken/extensions/active_job_adapter_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb