shoryuken 5.1.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d08b7287c9dcb2c488715f1313e11d9a9645c068d90fc160c76dbc57ff5d795
-  data.tar.gz: 1e6009538535c438a02267e948d5f3eacaedf4aa1283950733bfc51972dcf872
+  metadata.gz: 47b6564c21a283ca5d0cd71f7f59b36b9c99776a541b2a51c61a9fd5b590be08
+  data.tar.gz: cf94b4ba4d9b296e0af112704cbe0ecd201f57726c284c790351f79a23680c7a
 SHA512:
-  metadata.gz: dfc89030844c19828865a6495a4dcf4ef8e46f361303c4c906a0f1d56b2e9fdc46baebcd9eb6ee826a65a173dde4ed2d1a2e1ebc8dae64a8ed131fc5ebe65f13
-  data.tar.gz: af2ac18ce6d1445fd261344220b72271d78e10c4034bf565c782d4baad44f946459348342b1fbcbaa75fbe8984ef2655aad78cad2a7128feee5603466fc736d5
+  metadata.gz: d037b9887326e63ba1c8659a56e7707cf07b7a1a7d9f5027df7fb759d2896035335bc877487a346d7cac59e7b1916a8f5dc967e4431445b1e7ca9f6778282f7b
+  data.tar.gz: 4e0dd25a40a86a09ea87b0187d7f9f3d8525031b40830b47b29b434b688459e60209d7f8e4adb7d6c5deeedfb63b4139bceb0e38762b7fbf0dab884c1023063c

data/.devcontainer/Dockerfile

@@ -0,0 +1,17 @@
+# [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
+ARG VARIANT=2-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/ruby:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install additional gems.
+# RUN gem install <your-gem-names-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

data/.devcontainer/base.Dockerfile

@@ -0,0 +1,43 @@
+# [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
+ARG VARIANT=2-bullseye
+FROM ruby:${VARIANT}
+
+# Copy library scripts to execute
+COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/
+
+# [Option] Install zsh
+ARG INSTALL_ZSH="true"
+# [Option] Upgrade OS packages to their latest versions
+ARG UPGRADE_PACKAGES="true"
+# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
+    && apt-get purge -y imagemagick imagemagick-6-common \
+    # Install common packages, non-root user, rvm, core build tools
+    && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \
+    && bash /tmp/library-scripts/ruby-debian.sh "none" "${USERNAME}" "true" "true" \
+    && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+ENV NVM_DIR=/usr/local/share/nvm
+ENV NVM_SYMLINK_CURRENT=true \
+    PATH=${NVM_DIR}/current/bin:${PATH}
+RUN bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "${NODE_VERSION}" "${USERNAME}" \
+    && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+
+ # Remove library scripts for final image
+RUN rm -rf /tmp/library-scripts
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install additional gems.
+# RUN gem install <your-gem-names-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

data/.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.224.3/containers/ruby
+{
+  "name": "Ruby",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "args": {
+      // Update 'VARIANT' to pick a Ruby version: 3, 3.1, 3.0, 2, 2.7, 2.6
+      // Append -bullseye or -buster to pin to an OS version.
+      // Use -bullseye variants on local on arm64/Apple Silicon.
+      "VARIANT": "3-bullseye",
+      // Options
+      "NODE_VERSION": "none"
+    }
+  },
+
+  // Set *default* container specific settings.json values on container create.
+  "settings": {},
+
+  // Add the IDs of extensions you want installed when the container is created.
+  "extensions": ["rebornix.Ruby"],
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // Use 'postCreateCommand' to run commands after the container is created.
+  // "postCreateCommand": "ruby --version",
+
+  // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode",
+  "features": {
+    "github-cli": "latest",
+    "aws-cli": "latest"
+  }
+}

data/.github/workflows/specs.yml

@@ -9,9 +9,14 @@ jobs:
     name: All Specs
     strategy:
       matrix:
-        ruby: ['2.4.4', '2.5.1', '2.6.3']
-        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
+        gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
     runs-on: ubuntu-20.04
+    services:
+      moto_sqs:
+        image: quay.io/cjlarose/moto-sqs-server:1.1.0
+        ports:
+          - 5000:5000
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
     steps:
@@ -22,9 +27,9 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run specs
-        env:
-          SPEC_ALL: true
         run: bundle exec rake spec
+      - name: Run integration specs
+        run: bundle exec rake spec:integration
   rails_specs:
     name: Rails Specs
     strategy:
@@ -54,4 +59,4 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run Rails specs
-        run: bundle exec rake rails_specs
+        run: bundle exec rake spec:rails

data/.github/workflows/stale.yml

@@ -0,0 +1,20 @@
+name: "Close stale issues and PRs"
+on:
+  schedule:
+    - cron: "30 1 * * *" # At 01:30 - https://crontab.guru/#30_1_*_*_*
+  workflow_dispatch: {}
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v4
+        with:
+          stale-issue-message: This issue is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
+          stale-pr-message: This PR is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
+          close-issue-message: This issue was closed because it hasn't seen activity for a while.
+          close-pr-message: This PR was closed because it hasn't seen activity for a while.
+          days-before-stale: 60
+          days-before-close: 7

data/.gitignore

@@ -25,4 +25,4 @@ shoryuken.yml
 *.log
 .env
 rubocop.html
-.byebug_history
+.byebug_history

data/.rubocop.yml

@@ -23,7 +23,7 @@ Metrics/AbcSize:
   # because codeclimate already give that for us with more details
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 125
 
 Style/Alias:

data/Appraisals

@@ -1,3 +1,11 @@
+appraise 'aws_sdk_core_2' do
+  group :test do
+    gem 'aws-sdk-core', '~> 2'
+    gem 'webrick' # required for Ruby 3
+    remove_gem 'aws-sdk-sqs'
+  end
+end
+
 appraise 'rails_4_2' do
   group :test do
     gem 'activejob', '~> 4.2'

data/CHANGELOG.md

@@ -1,3 +1,60 @@
+## [v6.0.0] - 2022-02-18
+
+- Breaking changes: Initialize Rails before parsing config file
+  - [#686](https://github.com/ruby-shoryuken/shoryuken/pull/686)
+  - Previously, Shoryuken read its configuration from an optional YAML file, then allowed CLI arguments to override those, then initialized the Rails application (provided that `--rails` or `-R` was specified). This behavior meant that the config file did not have access to things like environment variables that were initialized by Rails (such as when using `dotenv`). With this change, Rails is initialized much earlier in the process. After Rails is initialized, the YAML configuration file is interpreted, and CLI arguments are finally interpreted last. Most applications will not need to undergo changes in order to upgrade, but the new load order could technically result in different behavior depending on the application's YAML configuration file or Rails initializers.
+
+## [v5.3.2] - 2022-01-19
+
+- (Bugfix) Preserve queue weights when unpausing queues
+  - [#687](https://github.com/ruby-shoryuken/shoryuken/pull/687)
+
+- Improve error message on startup when shoryuken has insufficient permissions to access a queue
+  - [#691](https://github.com/ruby-shoryuken/shoryuken/pull/691)
+
+## [v5.3.1] - 2022-01-07
+
+- (Bugfix) Fix issue where, when using the TSTP or USR1 signals for soft shutdowns, it was possible for shoryuken to terminate without first attempting to handle all messages it fetched from SQS
+  - [#676](https://github.com/ruby-shoryuken/shoryuken/pull/676)
+
+## [v5.3.0] - 2021-10-31
+
+- (Refactor) Use Forwardable within Message to avoid method boilerplate
+  - [#681](https://github.com/ruby-shoryuken/shoryuken/pull/681)
+
+- Add basic health check API
+  - [#679](https://github.com/ruby-shoryuken/shoryuken/pull/679)
+
+## [v5.2.3] - 2021-07-29
+
+- Fire new `:utilization_update` event any time a worker pool's utilization changes
+  - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
+
+## [v5.2.2] - 2021-06-22
+
+- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
+  - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
+
+## [v5.2.1] - 2021-04-06
+
+- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
+  - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
+
+- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
+  - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
+  - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
+
+- Remove test stub for `Concurrent.global_io_executor`
+  - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
+
+- Run integration tests on CI
+  - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
+
+## [v5.2.0] - 2021-02-26
+
+- Set `executions` correctly for ActiveJob jobs
+  - [#657](https://github.com/phstc/shoryuken/pull/657)
+
 ## [v5.1.1] - 2021-02-10
 
 - Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail

data/Gemfile

@@ -16,5 +16,5 @@ end
 group :development do
   gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
   gem 'pry-byebug', '3.9.0'
-  gem 'rubocop'
+  gem 'rubocop', '<= 1.12'
 end

data/README.md

@@ -69,7 +69,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ### Testing
 
-To run all specs against the latest dependency vesions, execute
+To run all unit specs against the latest dependency vesions, execute
 
 ```sh
 bundle exec rake spec
@@ -78,5 +78,11 @@ bundle exec rake spec
 To run all Rails-related specs against all supported versions of Rails, execute
 
 ```sh
-bundle exec appraisal rake rails_specs
+bundle exec appraisal rake spec:rails
+```
+
+To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+
+```sh
+bundle exec rake spec:integration
 ```

data/Rakefile

@@ -3,10 +3,21 @@ $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run Rails specs only'
+    RSpec::Core::RakeTask.new(:rails) do |t|
+      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    end
 
-  rails_task = RSpec::Core::RakeTask.new(:rails_specs)
-  rails_task.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    desc 'Run integration specs only'
+    RSpec::Core::RakeTask.new(:integration) do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+    end
+  end
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -4,6 +4,9 @@ require 'date'
 module Shoryuken
   module CLI
     class SQS < Base
+      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 256 * 1024
+
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
@@ -51,14 +54,56 @@ module Shoryuken
           end
         end
 
-        def batch_send(url, messages, messages_per_batch = 10)
-          messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
-            sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
-              say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+        def batch_send(url, messages, max_batch_size = 10)
+          messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
+          batch_send_normalized_messages url, messages, max_batch_size
+        end
+
+        def batch_send_normalized_messages(url, messages, max_batch_size)
+          # Repeatedly take the longest prefix of messages such that
+          # 1. The number of messages is less than or equal to max_batch_size
+          # 2. The total message payload size is less than or equal to the
+          #    batch payload limit
+          while messages.size.positive?
+            batch_size = max_batch_size
+            loop do
+              batch = messages.take batch_size
+
+              unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
+                batch_size = batch.size - 1
+                next
+              end
+
+              sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
+                say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+              end
+              messages = messages.drop batch.size
+              break
             end
           end
         end
 
+        def batch_payload_size(messages)
+          messages.sum(&method(:message_size))
+        end
+
+        def message_size(message)
+          attribute_size = (message[:message_attributes] || []).sum do |name, value|
+            name_size = name.to_s.bytesize
+            data_type_size = value[:data_type].bytesize
+            value_size = if value[:string_value]
+                           value[:string_value].bytesize
+                         elsif value[:binary_value]
+                           value[:binary_value].bytesize
+                         end
+            name_size + data_type_size + value_size
+          end
+
+          body_size = message[:message_body].bytesize
+
+          attribute_size + body_size
+        end
+
         def find_all(url, limit)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -160,7 +205,7 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/gemfiles/aws_sdk_core_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob"
+  gem "aws-sdk-core", "~> 2"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+  gem "webrick"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken/environment_loader.rb

@@ -18,12 +18,12 @@ module Shoryuken
     end
 
     def setup_options
+      initialize_rails if load_rails?
       initialize_options
       initialize_logger
     end
 
     def load
-      load_rails if Shoryuken.options[:rails]
       prefix_active_job_queue_names
       parse_queues
       require_workers
@@ -55,7 +55,7 @@ module Shoryuken
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
-    def load_rails
+    def initialize_rails
       # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
 
       require 'rails'
@@ -79,7 +79,13 @@ module Shoryuken
       end
     end
 
+    def load_rails?
+      options[:rails]
+    end
+
     def prefix_active_job_queue_name(queue_name, weight)
+      return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
+
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 
@@ -157,9 +163,17 @@ module Shoryuken
 
       return if non_existent_queues.none?
 
+      # NOTE: HEREDOC's ~ operator removes indents, but is only available Ruby 2.3+
+      # See github PR: https://github.com/ruby-shoryuken/shoryuken/pull/691#issuecomment-1007653595
+      error_msg = <<-MSG.gsub(/^\s+/, '')
+        The specified queue(s) #{non_existent_queues.join(', ')} do not exist.
+        Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
+        It's also possible that you don't have permission to access the specified queues.
+      MSG
+
       fail(
         ArgumentError,
-        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
+        error_msg
       )
     end
 

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -82,8 +82,10 @@ module ActiveJob
 
         shoryuken_options body_parser: :json, auto_delete: true
 
-        def perform(_sqs_msg, hash)
-          Base.execute hash
+        def perform(sqs_msg, hash)
+          receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+          past_receives = receive_count - 1
+          Base.execute hash.merge({ 'executions' => past_receives })
         end
       end
 

data/lib/shoryuken/launcher.rb

@@ -28,12 +28,30 @@ module Shoryuken
 
       initiate_stop
 
+      stop_new_dispatching
+      await_dispatching_in_progress
+
       executor.shutdown
       executor.wait_for_termination
     end
 
+    def healthy?
+      Shoryuken.groups.keys.all? do |group|
+        manager = @managers.find { |m| m.group == group }
+        manager && manager.running?
+      end
+    end
+
     private
 
+    def stop_new_dispatching
+      @managers.each(&:stop_new_dispatching)
+    end
+
+    def await_dispatching_in_progress
+      @managers.each(&:await_dispatching_in_progress)
+    end
+
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
@@ -71,6 +89,7 @@ module Shoryuken
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
+          group,
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],

data/lib/shoryuken/manager.rb

@@ -6,27 +6,47 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency, executor)
-      @fetcher          = fetcher
-      @polling_strategy = polling_strategy
-      @max_processors   = concurrency
-      @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @executor         = executor
-      @running          = Concurrent::AtomicBoolean.new(true)
+    attr_reader :group
+
+    def initialize(group, fetcher, polling_strategy, concurrency, executor)
+      @group                      = group
+      @fetcher                    = fetcher
+      @polling_strategy           = polling_strategy
+      @max_processors             = concurrency
+      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @executor                   = executor
+      @running                    = Concurrent::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @dispatching_release_signal = ::Queue.new
     end
 
     def start
+      fire_utilization_update_event
       dispatch_loop
     end
 
-    private
+    def stop_new_dispatching
+      @stop_new_dispatching.make_true
+    end
+
+    def await_dispatching_in_progress
+      # There might still be a dispatching on-going, as the response from SQS could take some time
+      # We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
+      # We use a queue, as the dispatch_loop is running on another thread, and this is a efficient way of communicating between threads.
+      @dispatching_release_signal.pop
+    end
 
     def running?
       @running.true? && @executor.running?
     end
 
+    private
+
     def dispatch_loop
-      return unless running?
+      if @stop_new_dispatching.true? || !running?
+        @dispatching_release_signal << 1
+        return
+      end
 
       @executor.post { dispatch }
     end
@@ -59,6 +79,8 @@ module Shoryuken
 
     def processor_done(queue)
       @busy_processors.decrement
+      fire_utilization_update_event
+
       client_queue = Shoryuken::Client.queues(queue)
       return unless client_queue.fifo?
       return unless @polling_strategy.respond_to?(:message_processed)
@@ -72,6 +94,7 @@ module Shoryuken
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
+      fire_utilization_update_event
 
       Concurrent::Promise
         .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
@@ -87,7 +110,6 @@ module Shoryuken
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
-
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -114,5 +136,13 @@ module Shoryuken
 
       @running.make_false
     end
+
+    def fire_utilization_update_event
+      fire_event :utilization_update, false, {
+        group: @group,
+        max_processors: @max_processors,
+        busy_processors: busy
+      }
+    end
   end
 end

data/lib/shoryuken/message.rb

@@ -1,5 +1,16 @@
 module Shoryuken
   class Message
+    extend Forwardable
+
+    def_delegators(:data,
+                   :message_id,
+                   :receipt_handle,
+                   :md5_of_body,
+                   :body,
+                   :attributes,
+                   :md5_of_message_attributes,
+                   :message_attributes)
+
     attr_accessor :client, :queue_url, :queue_name, :data
 
     def initialize(client, queue, data)
@@ -29,33 +40,5 @@ module Shoryuken
         visibility_timeout: timeout
       )
     end
-
-    def message_id
-      data.message_id
-    end
-
-    def receipt_handle
-      data.receipt_handle
-    end
-
-    def md5_of_body
-      data.md5_of_body
-    end
-
-    def body
-      data.body
-    end
-
-    def attributes
-      data.attributes
-    end
-
-    def md5_of_message_attributes
-      data.md5_of_message_attributes
-    end
-
-    def message_attributes
-      data.message_attributes
-    end
   end
 end