shoryuken 5.0.0 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce36f4dac1c75e383185df15db196aec57c968dda1a5823b8d944ad4a9675906
-  data.tar.gz: a460a585e97a6c0a3432f6b28bec254d494900672a06bffbd7461e830a8de0cf
+  metadata.gz: 313fe407e7c7715e8a83bd3fcef713cb5755d31e9d94345fe26157c823f4b156
+  data.tar.gz: 53890b5dc5dcf0d12fd5153b459d9cb317dace747d95516f754bd320be7eb105
 SHA512:
-  metadata.gz: 833bab295345b25456f617f36e4ec2f2cfd0d823a01ddb87d8f528597ab79b91ce0ae8c67b47ffc2a9d1400f0a5cf62df3125fbdfbb49abd3a5e4a5eb3b7b0ee
-  data.tar.gz: 91a4355cd9e180f006e8768bde9627c1df9cf0057d294bdfcb422f479de2ef1784aa5b904c7b97dac8ed7bb7b9879bdea56eb4cd00d51ff456d3b4555f88c991
+  metadata.gz: b349de0985d3beac244ed9f1b8886c7b71e18a810936d1cf1e89baddff507ee5b83af100a598653976ba8e337f0e8ad25f3db6b178b6bbd544e9696772b63160
+  data.tar.gz: 92ef7d6d5648485c1556225a81e6808d6ba127342e3edcb93cacb10e4f1070077f40f6b5ec3b9a1809226e092eb99c2710c2d238c4f16ea3072455f83c2eb643

data/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: phstc
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

data/.github/workflows/specs.yml

@@ -0,0 +1,57 @@
+name: Specs
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  all_specs:
+    name: All Specs
+    strategy:
+      matrix:
+        ruby: ['2.4.4', '2.5.1', '2.6.3']
+        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+    runs-on: ubuntu-20.04
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run specs
+        env:
+          SPEC_ALL: true
+        run: bundle exec rake spec
+  rails_specs:
+    name: Rails Specs
+    strategy:
+      matrix:
+        rails: ['4.2', '5.2', '6.0', '6.1']
+        include:
+          - rails: '4.2'
+            ruby: '2.2'
+            gemfile: gemfiles/rails_4_2.gemfile
+          - rails: '5.2'
+            ruby: '2.5'
+            gemfile: gemfiles/rails_5_2.gemfile
+          - rails: '6.0'
+            ruby: '2.6'
+            gemfile: gemfiles/rails_6_0.gemfile
+          - rails: '6.1'
+            ruby: '3.0'
+            gemfile: gemfiles/rails_6_1.gemfile
+    runs-on: ubuntu-20.04
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run Rails specs
+        run: bundle exec rake rails_specs

data/.reek.yml

@@ -0,0 +1,5 @@
+---
+detectors:
+
+  UtilityFunction:
+    public_methods_only: true

data/.rubocop.yml

@@ -1,7 +1,7 @@
 AllCops:
   Exclude:
     - '**/Gemfile'
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
 
 Metrics/PerceivedComplexity:
   Enabled: false
@@ -32,7 +32,7 @@ Style/Alias:
 Style/PerlBackrefs:
   Enabled: false
 
-Layout/TrailingBlankLines:
+Layout/TrailingEmptyLines:
   Enabled: false
 
 # Override the HoundCI custom rules (they do not use Rubocop defaults)
@@ -76,7 +76,7 @@ Style/GuardClause:
 Style/RegexpLiteral:
   Enabled: false
 
-Lint/HandleExceptions:
+Lint/SuppressedException:
   Enabled: false
 
 Lint/AssignmentInCondition:
@@ -117,6 +117,3 @@ Security/YAMLLoad:
 
 Naming/MemoizedInstanceVariableName:
   Enabled: false
-
-Performance/RedundantBlockCall:
-  Enabled: false

data/Appraisals

@@ -0,0 +1,28 @@
+appraise 'rails_4_2' do
+  group :test do
+    gem 'activejob', '~> 4.2'
+  end
+
+  group :development do
+    gem 'appraisal', '~> 2.2'
+    remove_gem 'pry-byebug'
+  end
+end
+
+appraise 'rails_5_2' do
+  group :test do
+    gem 'activejob', '~> 5.2'
+  end
+end
+
+appraise 'rails_6_0' do
+  group :test do
+    gem 'activejob', '~> 6.0'
+  end
+end
+
+appraise 'rails_6_1' do
+  group :test do
+    gem 'activejob', '~> 6.1'
+  end
+end

data/CHANGELOG.md

@@ -1,3 +1,52 @@
+## [v5.1.0] - 2021-02-06
+
+- Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
+  - [#635](https://github.com/phstc/shoryuken/pull/635)
+  - [#648](https://github.com/phstc/shoryuken/pull/648)
+  - [#651](https://github.com/phstc/shoryuken/pull/651)
+
+- Unpause FIFO queues on worker completion
+  - [#644](https://github.com/phstc/shoryuken/pull/644)
+
+- Add multiple versions of Rails to test matrix
+  - [#647](https://github.com/phstc/shoryuken/pull/647)
+
+- Migrate from Travis CI to Github Actions
+  - [#649](https://github.com/phstc/shoryuken/pull/649)
+  - [#650](https://github.com/phstc/shoryuken/pull/650)
+  - [#652](https://github.com/phstc/shoryuken/pull/652)
+
+## [v5.0.6] - 2020-12-30
+
+- Load ShoryukenConcurrentSendAdapter when loading Rails
+  - [#642](https://github.com/phstc/shoryuken/pull/642)
+
+## [v5.0.5] - 2020-06-07
+
+- Add ability to configure queue by ARN
+  - [#603](https://github.com/phstc/shoryuken/pull/603)
+
+## [v5.0.4] - 2020-02-20
+
+- Add endpoint option to SQS CLI
+  - [#595](https://github.com/phstc/shoryuken/pull/595)
+
+## [v5.0.3] - 2019-11-30
+
+- Add support for sending messages asynchronous with Active Job using `shoryuken_concurrent_send`
+  - [#589](https://github.com/phstc/shoryuken/pull/589)
+  - [#588](https://github.com/phstc/shoryuken/pull/588)
+
+## [v5.0.2] - 2019-11-02
+
+- Fix Queue order is reversed if passed through CLI
+  - [#571](https://github.com/phstc/shoryuken/pull/583)
+
+## [v5.0.1] - 2019-06-19
+
+- Add back attr_accessor for `stop_callback`
+  - [#571](https://github.com/phstc/shoryuken/pull/571)
+
 ## [v5.0.0] - 2019-06-18
 
 - Fix bug where empty queues were not paused in batch processing mode

data/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
-  gem 'activejob', '~> 4'
+  gem 'activejob'
   gem 'aws-sdk-core', '~> 3'
   gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
@@ -14,5 +14,7 @@ group :test do
 end
 
 group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug', '3.9.0'
   gem 'rubocop'
 end

data/README.md

@@ -1,10 +1,12 @@
+**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? Fill up this form https://forms.gle/8kTso8ixa9Sfp6rJ9**
+
 # Shoryuken
 
 ![Shoryuken](shoryuken.jpg)
 
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
-[![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
+[![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
 [![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
 
 ## Key features
@@ -21,7 +23,7 @@ Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com
 
 ## Requirements
 
-Ruby 2.0 or greater.
+Ruby 2.4 or greater.
 
 ## Installation
 
@@ -37,7 +39,7 @@ If you are using AWS SDK version 3, please also add this line:
 gem 'aws-sdk-sqs'
 ```
 
-The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3. 
+The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
 
 And then execute:
 
@@ -64,3 +66,17 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+### Testing
+
+To run all specs against the latest dependency vesions, execute
+
+```sh
+bundle exec rake spec
+```
+
+To run all Rails-related specs against all supported versions of Rails, execute
+
+```sh
+bundle exec appraisal rake rails_specs
+```

data/Rakefile

@@ -4,6 +4,9 @@ $stdout.sync = true
 begin
   require 'rspec/core/rake_task'
   RSpec::Core::RakeTask.new(:spec)
+
+  rails_task = RSpec::Core::RakeTask.new(:rails_specs)
+  rails_task.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -5,6 +5,7 @@ module Shoryuken
   module CLI
     class SQS < Base
       namespace :sqs
+      class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
       no_commands do
         def normalize_dump_message(message)
@@ -19,8 +20,15 @@ module Shoryuken
           }
         end
 
+        def client_options
+          endpoint = options[:endpoint]
+          {}.tap do |hash|
+            hash[:endpoint] = endpoint unless endpoint.to_s.empty?
+          end
+        end
+
         def sqs
-          @_sqs ||= Aws::SQS::Client.new
+          @_sqs ||= Aws::SQS::Client.new(client_options)
         end
 
         def find_queue_url(queue_name)

data/gemfiles/.gitignore

@@ -0,0 +1 @@
+*.gemfile.lock

data/gemfiles/rails_4_2.gemfile

@@ -0,0 +1,20 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 4.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", "~> 2.2"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_5_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 5.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.1"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken/default_worker_registry.rb

@@ -1,7 +1,7 @@
 module Shoryuken
   class DefaultWorkerRegistry < WorkerRegistry
     def initialize
-      @workers = {}
+      @workers = Concurrent::Hash.new
     end
 
     def batch_receive_messages?(queue)

data/lib/shoryuken/environment_loader.rb

@@ -20,7 +20,6 @@ module Shoryuken
     def setup_options
       initialize_options
       initialize_logger
-      merge_cli_defined_queues
     end
 
     def load
@@ -71,22 +70,15 @@ module Shoryuken
             ::Rails.application.config.eager_load = true
           end
         end
-        require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?
+        if Shoryuken.active_job?
+          require 'shoryuken/extensions/active_job_extensions'
+          require 'shoryuken/extensions/active_job_adapter'
+          require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+        end
         require File.expand_path('config/environment.rb')
       end
     end
 
-    def merge_cli_defined_queues
-      cli_defined_queues = options[:queues].to_a
-
-      cli_defined_queues.each do |cli_defined_queue|
-        # CLI defined queues override config_file defined queues
-        Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
-
-        Shoryuken.options[:queues] << cli_defined_queue
-      end
-    end
-
     def prefix_active_job_queue_name(queue_name, weight)
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -33,8 +33,12 @@ module ActiveJob
       def enqueue(job, options = {}) #:nodoc:
         register_worker!(job)
 
+        job.sqs_send_message_parameters.merge! options
+
         queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(queue, job, options))
+        send_message_params = message queue, job
+        job.sqs_send_message_parameters = send_message_params
+        queue.send_message send_message_params
       end
 
       def enqueue_at(job, timestamp) #:nodoc:
@@ -50,35 +54,29 @@ module ActiveJob
         delay
       end
 
-      def message(queue, job, options = {})
+      def message(queue, job)
         body = job.serialize
+        job_params = job.sqs_send_message_parameters
+
+        attributes = job_params[:message_attributes] || {}
 
-        msg = {}
+        msg = {
+          message_body: body,
+          message_attributes: attributes.merge(MESSAGE_ATTRIBUTES)
+        }
 
         if queue.fifo?
           # See https://github.com/phstc/shoryuken/issues/457
           msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
         end
 
-        msg[:message_body] = body
-        msg[:message_attributes] = message_attributes
-
-        msg.merge(options)
+        msg.merge(job_params.except(:message_attributes))
       end
 
       def register_worker!(job)
         Shoryuken.register_worker(job.queue_name, JobWrapper)
       end
 
-      def message_attributes
-        @message_attributes ||= {
-          'shoryuken_class' => {
-            string_value: JobWrapper.to_s,
-            data_type: 'String'
-          }
-        }
-      end
-
       class JobWrapper #:nodoc:
         include Shoryuken::Worker
 
@@ -88,6 +86,13 @@ module ActiveJob
           Base.execute hash
         end
       end
+
+      MESSAGE_ATTRIBUTES = {
+        'shoryuken_class' => {
+          string_value: JobWrapper.to_s,
+          data_type: 'String'
+        }
+      }.freeze
     end
   end
 end

data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb

@@ -0,0 +1,50 @@
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+module ActiveJob
+  module QueueAdapters
+    # == Shoryuken concurrent adapter for Active Job
+    #
+    # This adapter sends messages asynchronously (ie non-blocking) and allows
+    # the caller to set up handlers for both success and failure
+    #
+    # To use this adapter, set up as:
+    #
+    # success_handler = ->(response, job, options) { StatsD.increment("#{job.class.name}.success") }
+    # error_handler = ->(err, job, options) { StatsD.increment("#{job.class.name}.failure") }
+    #
+    # adapter = ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter.new(success_handler, error_handler)
+    #
+    # config.active_job.queue_adapter = adapter
+    class ShoryukenConcurrentSendAdapter < ShoryukenAdapter
+      def initialize(success_handler = nil, error_handler = nil)
+        @success_handler = success_handler
+        @error_handler = error_handler
+      end
+
+      def enqueue(job, options = {})
+        send_concurrently(job, options) { |f_job, f_options| super(f_job, f_options) }
+      end
+
+      def success_handler
+        @success_handler ||= ->(_send_message_response, _job, _options) { nil }
+      end
+
+      def error_handler
+        @error_handler ||= begin
+          lambda { |error, job, _options|
+            Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
+          }
+        end
+      end
+
+      private
+
+      def send_concurrently(job, options)
+        Concurrent::Promises
+          .future(job, options) { |f_job, f_options| [yield(f_job, f_options), f_job, f_options] }
+          .then { |send_message_response, f_job, f_options| success_handler.call(send_message_response, f_job, f_options) }
+          .rescue(job, options) { |err, f_job, f_options| error_handler.call(err, f_job, f_options) }
+      end
+    end
+  end
+end

data/lib/shoryuken/extensions/active_job_extensions.rb

@@ -0,0 +1,37 @@
+module Shoryuken
+  module ActiveJobExtensions
+    # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs
+    # (instances of ActiveJob::Base). Shoryuken ActiveJob queue adapters use
+    # these parameters when enqueueing jobs; other adapters can ignore them.
+    module SQSSendMessageParametersAccessor
+      extend ActiveSupport::Concern
+
+      included do
+        attr_accessor :sqs_send_message_parameters
+      end
+    end
+
+    # Initializes SQS SendMessage parameters on instances of ActiveJobe::Base
+    # to the empty hash, and populates it whenever `#enqueue` is called, such
+    # as when using ActiveJob::Base.set.
+    module SQSSendMessageParametersSupport
+      def initialize(*arguments)
+        super(*arguments)
+        self.sqs_send_message_parameters = {}
+      end
+
+      def enqueue(options = {})
+        sqs_options = options.extract! :message_attributes,
+                                       :message_system_attributes,
+                                       :message_deduplication_id,
+                                       :message_group_id
+        sqs_send_message_parameters.merge! sqs_options
+
+        super
+      end
+    end
+  end
+end
+
+ActiveJob::Base.include Shoryuken::ActiveJobExtensions::SQSSendMessageParametersAccessor
+ActiveJob::Base.prepend Shoryuken::ActiveJobExtensions::SQSSendMessageParametersSupport

data/lib/shoryuken/manager.rb

@@ -57,8 +57,13 @@ module Shoryuken
       @max_processors - busy
     end
 
-    def processor_done
+    def processor_done(queue)
       @busy_processors.decrement
+      client_queue = Shoryuken::Client.queues(queue)
+      return unless client_queue.fifo?
+      return unless @polling_strategy.respond_to?(:message_processed)
+
+      @polling_strategy.message_processed(queue)
     end
 
     def assign(queue_name, sqs_msg)
@@ -68,9 +73,10 @@ module Shoryuken
 
       @busy_processors.increment
 
-      Concurrent::Promise.execute(
-        executor: @executor
-      ) { Processor.process(queue_name, sqs_msg) }.then { processor_done }.rescue { processor_done }
+      Concurrent::Promise
+        .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
+        .then { processor_done(queue_name) }
+        .rescue { processor_done(queue_name) }
     end
 
     def dispatch_batch(queue)

data/lib/shoryuken/options.rb

@@ -14,9 +14,11 @@ module Shoryuken
       }
     }.freeze
 
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :default_worker_options, :groups,
-                  :launcher_executor, :sqs_client, :sqs_client_receive_message_opts,
-                  :start_callback, :worker_executor, :worker_registry
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
+                  :launcher_executor,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry
+    attr_writer :default_worker_options, :sqs_client
+    attr_reader :sqs_client_receive_message_opts
 
     def initialize
       self.groups = {}

data/lib/shoryuken/polling/base.rb

@@ -40,6 +40,8 @@ module Shoryuken
         fail NotImplementedError
       end
 
+      def message_processed(_queue); end
+
       def active_queues
         fail NotImplementedError
       end