shoryuken 3.0.4 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1367b27e316c258563ef328b78bc64947c849d8
-  data.tar.gz: 285db9748d737bfe07a6904db381d3c11883cd05
+  metadata.gz: 779c3e9c7f708f0a6dccb3e8b0c9f7e2490b25e1
+  data.tar.gz: bf260a2793ff8c07ddfb670832ba5f12af8d85d9
 SHA512:
-  metadata.gz: 7f85154f339098c0b04a01673548b0c688217290772131dc28a1ed0c378b0cb671576f5a195143d915a8ab70ed836b089bd12ff6c4d6aa78d81bf858729323a6
-  data.tar.gz: 9b5180a9baf239178c11afcc7f55d31392162deeb95aa7f49836a64e94fcc114b145ba82414b83ad22d278612d3cca53436bb804d41d98ccfbe9f1eea4f678cc
+  metadata.gz: 1f3e7a4b668c929daa1b5424b7a8ab7f028238b3f6dfe0e812351991bbc37923b58491760df6e4ed4921624cd4954761daeac257c6a540fd193d160d121044de
+  data.tar.gz: 02f2bf2e813cbe1624ae83d16936196ecf523fe8d3773f9b0c39813aa0822d3c71fd273ee01a63072c20d86bf05f4c1a775eda3fae41cec4bf55ea567f772aef

data/.rubocop.yml

@@ -32,7 +32,7 @@ Metrics/ParameterLists:
   Enabled: false
 
 Metrics/LineLength:
-  Max: 120
+  Max: 130
 
 Metrics/MethodLength :
   Enabled: false

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   - 2.1.0
   - 2.2.0
   - 2.3.3
+  - 2.4.1
 
 notifications:
   email:

data/CHANGELOG.md

@@ -1,4 +1,75 @@
+## [v3.1.0] - 2017-07-02
+
+- Add shoryuken sqs delete command
+ - [#395](https://github.com/phstc/shoryuken/pull/395)
+
+- Add processing groups support; Concurrency per queue support
+ - [#389](https://github.com/phstc/shoryuken/pull/389)
+
+- Terminate Shoryuken if the fetcher crashes
+ - [#389](https://github.com/phstc/shoryuken/pull/389)
+
+## [v3.0.11] - 2017-06-24
+
+- Add shoryuken sqs create command
+ - [#388](https://github.com/phstc/shoryuken/pull/388)
+
+## [v3.0.10] - 2017-06-24
+
+- Allow aws sdk v3
+ - [#381](https://github.com/phstc/shoryuken/pull/381)
+
+- Allow configuring Rails via the config file
+ - [#387](https://github.com/phstc/shoryuken/pull/387)
+
+## [v3.0.9] - 2017-06-05
+
+- Allow configuring queue URLs instead of names
+ - [#378](https://github.com/phstc/shoryuken/pull/378)
+
+## [v3.0.8] - 2017-06-02
+
+- Fix miss handling empty batch fetches
+ - [#376](https://github.com/phstc/shoryuken/pull/376)
+
+- Various minor styling changes :lipstick:
+ - [#373](https://github.com/phstc/shoryuken/pull/373)
+
+- Logout when batch delete returns any failure
+ - [#371](https://github.com/phstc/shoryuken/pull/371)
+
+## [v3.0.7] - 2017-05-18
+
+- Trigger events for dispatch
+ - [#362](https://github.com/phstc/shoryuken/pull/362)
+
+- Log (warn) exponential backoff tries
+ - [#365](https://github.com/phstc/shoryuken/pull/365)
+
+- Fix displaying of long queue names in `shoryuken sqs ls`
+ - [#366](https://github.com/phstc/shoryuken/pull/366)
+
+## [v3.0.6] - 2017-04-11
+
+- Fix delay option type
+ - [#356](https://github.com/phstc/shoryuken/pull/356)
+
+## [v3.0.5] - 2017-04-09
+
+- Pause endless dispatcher to avoid CPU overload
+ - [#354](https://github.com/phstc/shoryuken/pull/354)
+
+- Auto log processor errors
+ - [#355](https://github.com/phstc/shoryuken/pull/355)
+
+- Add a delay as a CLI param
+ - [#350](https://github.com/phstc/shoryuken/pull/350)
+
+- Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
+ - [#344](https://github.com/phstc/shoryuken/pull/344)
+
 ## [v3.0.4] - 2017-03-24
+
 - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
  - [#344](https://github.com/phstc/shoryuken/pull/344)
 
@@ -6,18 +77,22 @@
  - [#345](https://github.com/phstc/shoryuken/pull/345)
 
 ## [v3.0.3] - 2017-03-19
+
 - Update `sqs` CLI commands to use `get_queue_url` when appropriated
  - [#341](https://github.com/phstc/shoryuken/pull/341)
 
 ## [v3.0.2] - 2017-03-19
+
 - Fix custom SQS client initialization
  - [#335](https://github.com/phstc/shoryuken/pull/335)
 
 ## [v3.0.1] - 2017-03-13
+
 - Fix commands sqs mv and dump `options.delete` checker
  - [#332](https://github.com/phstc/shoryuken/pull/332)
 
 ## [v3.0.0] - 2017-03-12
+
 - Replace Celluloid with Concurrent Ruby
  - [#291](https://github.com/phstc/shoryuken/pull/291)
 
@@ -37,6 +112,7 @@
  - [#330](https://github.com/phstc/shoryuken/pull/330)
 
 ## [v2.1.3] - 2017-01-27
+
 - Show a warn message when batch isn't supported
  - [#302](https://github.com/phstc/shoryuken/pull/302)
 
@@ -47,6 +123,7 @@
  - [#307](https://github.com/phstc/shoryuken/pull/307)
 
 ## [v2.1.2] - 2016-12-22
+
 - Fix loading `logfile` from shoryuken.yml
  - [#296](https://github.com/phstc/shoryuken/pull/296)
 
@@ -66,10 +143,12 @@
  - [#284](https://github.com/phstc/shoryuken/pull/284)
 
 ## [v2.1.1] - 2016-12-05
+
 - Fix aws deprecation warning message
  - [#279](https://github.com/phstc/shoryuken/pull/279)
 
 ## [v2.1.0] - 2016-12-03
+
 - Fix celluloid "running in BACKPORTED mode" warning
  - [#260](https://github.com/phstc/shoryuken/pull/260)
 

data/Gemfile

@@ -7,4 +7,5 @@ group :test do
   gem 'codeclimate-test-reporter', require: nil
   gem 'simplecov'
   gem 'multi_xml'
+  gem 'httparty'
 end

data/README.md

@@ -1,42 +1,23 @@
 # Shoryuken
 
-![](shoryuken.jpg)
+![Shoryuken](shoryuken.jpg)
 
-Shoryuken _sho-ryu-ken_ is a super-efficient [AWS SQS](https://aws.amazon.com/sqs/) thread-based message processor.
+Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
 [![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
 
 ## Key features
 
-### Load balancing
-
-Yeah, Shoryuken load balances the messages consumption!
-
-Given this configuration:
-
-```yaml
-concurrency: 25
-delay: 25
-queues:
-  - [high_priority, 6]
-  - [normal_priority, 2]
-  - [low_priority, 1]
-```
-
-And supposing all the queues are full of messages, the configuration above will make Shoryuken to process `high_priority` 3 times more than `normal_priority` and 6 times more than `low_priority`,
-splitting the work load among all available processors `concurrency: 25` .
-
-If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but only to process `normal_priority` and `low_priority`.
-
-If `high_priority` receives a new message, Shoryuken will smoothly increase back its weight one by one until it reaches the weight of 6 again.
-
-[If a queue gets empty, Shoryuken will pause checking it for `delay: 25`](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
-
-
-### Fetch in batches
-
-To be even more performant and cost effective, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
+- [Rails Active Job](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job)
+- [Queue Load balancing](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#load-balancing)
+- [Concurrency per queue](https://github.com/phstc/shoryuken/wiki/Processing-Groups)
+- [Long Polling](https://github.com/phstc/shoryuken/wiki/Long-Polling)
+- [Batch processing](https://github.com/phstc/shoryuken/wiki/Worker-options#batch)
+- [Auto extend visibility timeout](https://github.com/phstc/shoryuken/wiki/Worker-options#auto_visibility_timeout)
+- [Exponential backoff](https://github.com/phstc/shoryuken/wiki/Worker-options#retry_intervals)
+- [Middleware support](https://github.com/phstc/shoryuken/wiki/Middleware)
+- Amazon SQS CLI. See `shoryuken help sqs`
 
 ## Requirements
 
@@ -50,102 +31,19 @@ Add this line to your application's Gemfile:
 gem 'shoryuken'
 ```
 
-Or to get the latest updates:
-
-```ruby
-gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
-```
-
 And then execute:
 
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install shoryuken
-
-## Usage
-
-### Worker class
-
-```ruby
-class MyWorker
-  include Shoryuken::Worker
-
-  shoryuken_options queue: 'default', auto_delete: true
-  # shoryuken_options queue: ->{ "#{ENV['environment']}_default" }
-
-  # shoryuken_options body_parser: :json
-  # shoryuken_options body_parser: ->(sqs_msg){ REXML::Document.new(sqs_msg.body) }
-  # shoryuken_options body_parser: JSON
-
-  def perform(sqs_msg, body)
-    puts body
-  end
-end
-```
-
-[Check the Worker options documention](https://github.com/phstc/shoryuken/wiki/Worker-options).
-
-### Sending a message
-
-[Check the Sending a message documentation](https://github.com/phstc/shoryuken/wiki/Sending-a-message)
-
-### Middleware
-
-```ruby
-class MyMiddleware
-  def call(worker_instance, queue, sqs_msg, body)
-    puts 'Before work'
-    yield
-    puts 'After work'
-  end
-end
-```
-
-[Check the Middleware documentation](https://github.com/phstc/shoryuken/wiki/Middleware).
-
-### Shoryuken Configuration
-
-Sample configuration file `shoryuken.yml`.
-
-```yaml
-concurrency: 25  # The number of allocated threads to process messages. Default 25
-delay: 25        # The delay in seconds to pause a queue when it's empty. Default 0
-queues:
-  - [high_priority, 6]
-  - [normal_priority, 2]
-  - [low_priority, 1]
-```
-
-#### AWS Configuration
-
-[Check the Configure AWS Client documentation](https://github.com/phstc/shoryuken/wiki/Configure-the-AWS-Client)
-
-### Rails Integration
-
-[Check the Rails Integration Active Job documention](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job).
-
-### Start Shoryuken
-
 ```shell
-bundle exec shoryuken -r worker.rb -C shoryuken.yml
+$ bundle
 ```
 
-For other options check `bundle exec shoryuken help start`
-
-#### SQS commands
-
-Check also some available SQS commands `bundle exec shoryuken help sqs`, such as:
+## Usage
 
-- `ls` list queues
-- `mv` move messages from one queue to another
-- `dump` dump messages from a queue into a JSON lines file
-- `requeue` requeue messages from a dump file
+Check the [Getting Started](https://github.com/phstc/shoryuken/wiki/Getting-Started) page.
 
 ## More Information
 
-For more information on advanced topics such as signals (shutdown), ActiveJob integration, and so on please check the [Shoryuken Wiki](https://github.com/phstc/shoryuken/wiki).
+For more information check the [wiki page](https://github.com/phstc/shoryuken/wiki).
 
 ## Credits
 

data/bin/cli/base.rb

@@ -25,10 +25,8 @@ module Shoryuken
         end
 
         def print_format_column(column, size)
-          size = 40 if size > 40
           size_with_padding = size + 4
           column = column.to_s.ljust(size_with_padding)
-          column = "#{column[0...size - 2]}.." if column.size > size_with_padding
           column
         end
 

data/bin/cli/sqs.rb

@@ -32,7 +32,10 @@ module Shoryuken
               queue_url: url,
               entries: batch.map { |message| { id: message.message_id, receipt_handle: message.receipt_handle } }
             ).failed.any? do |failure|
-              say "Could not delete #{failure.id}, code: #{failure.code}", :yellow
+              say(
+                "Could not delete #{failure.id}, code: #{failure.code}, message: #{failure.message}, sender_fault: #{failure.sender_fault}",
+                :yellow
+              )
             end
           end
         end
@@ -182,6 +185,20 @@ module Shoryuken
 
         say "Purge request sent for #{queue_name}. The message deletion process takes up to 60 seconds", :yellow
       end
+
+      desc 'create QUEUE-NAME', 'Create a queue'
+      def create(queue_name)
+        queue_url = sqs.create_queue(queue_name: queue_name).queue_url
+
+        say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
+      end
+
+      desc 'delete QUEUE-NAME', 'delete a queue'
+      def delete(queue_name)
+        sqs.delete_queue(queue_url: find_queue_url(queue_name))
+
+        say "Queue #{queue_name} was successfully delete", :green
+      end
     end
   end
 end

data/bin/shoryuken

@@ -21,14 +21,22 @@ module Shoryuken
       method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
       method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
       method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
-      method_option :config_file, aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
       method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
       method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
       method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
       method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
+      method_option :delay,       aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
       def start
         opts = options.to_h.symbolize_keys
 
+        if opts[:config_file]
+          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+        end
+
+        opts[:config_file] = opts.delete(:config) if opts[:config]
+
         # Keep compatibility with old CLI queue format
         opts[:queues] = options[:queues].map { |q| q.split(',') } if options[:queues]
 

data/examples/default_worker.rb

@@ -4,6 +4,6 @@ class DefaultWorker
   shoryuken_options queue: 'default', auto_delete: true
 
   def perform(sqs_msg, body)
-    Shoryuken.logger.debug("Received message: '#{body}'")
+    Shoryuken.logger.debug("Received message: #{body}")
   end
 end

data/lib/shoryuken/default_worker_registry.rb

@@ -24,7 +24,7 @@ module Shoryuken
                        @workers[queue]
                      end
 
-      worker_class.new
+      worker_class.new if worker_class
     end
 
     def queues
@@ -34,7 +34,7 @@ module Shoryuken
     def register_worker(queue, clazz)
       if (worker_class = @workers[queue])
         if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          fail ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+          fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
             "because #{worker_class} is already registered for this queue, "\
             "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
         end

data/lib/shoryuken/environment_loader.rb

@@ -24,7 +24,7 @@ module Shoryuken
     end
 
     def load
-      load_rails if options[:rails]
+      load_rails if Shoryuken.options[:rails]
       prefix_active_job_queue_names
       parse_queues
       require_workers
@@ -42,9 +42,13 @@ module Shoryuken
     def config_file_options
       return {} unless (path = options[:config_file])
 
-      fail ArgumentError, "The supplied config file '#{path}' does not exist" unless File.exist?(path)
+      fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
-      YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
+      if result = YAML.load(ERB.new(IO.read(path)).result)
+        result.deep_symbolize_keys
+      else
+        {}
+      end
     end
 
     def initialize_logger
@@ -96,13 +100,24 @@ module Shoryuken
       end
     end
 
-    def parse_queue(queue, weight = nil)
-      Shoryuken.add_queue(queue, [weight.to_i, 1].max)
+    def parse_queue(queue, weight = nil, group)
+      Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
     end
 
     def parse_queues
-      Shoryuken.options[:queues].to_a.each do |queue_and_weight|
-        parse_queue(*queue_and_weight)
+      if Shoryuken.options[:queues].to_a.any?
+        Shoryuken.add_group('default', Shoryuken.options.fetch(:concurrency, 25))
+
+        Shoryuken.options[:queues].to_a.each do |queue, weight|
+          parse_queue(queue, weight, 'default')
+        end
+      end
+
+      Shoryuken.options[:groups].to_a.each do |group, options|
+        Shoryuken.add_group(group, options.fetch(:concurrency, 25))
+        options[:queues].to_a.each do |queue, weight|
+          parse_queue(queue, weight, group)
+        end
       end
     end
 
@@ -119,29 +134,34 @@ module Shoryuken
     end
 
     def validate_queues
-      Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
+      return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.ungrouped_queues.empty?
 
       non_existent_queues = []
 
-      Shoryuken.queues.uniq.each do |queue|
+      Shoryuken.ungrouped_queues.uniq.each do |queue|
         begin
           Shoryuken::Client.queues(queue)
-        rescue Aws::SQS::Errors::NonExistentQueue
+        rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
           non_existent_queues << queue
         end
       end
 
-      fail ArgumentError, "The specified queue(s) #{non_existent_queues} do not exist" if non_existent_queues.any?
+      return if non_existent_queues.none?
+
+      fail(
+        ArgumentError,
+        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
+      )
     end
 
     def validate_workers
       return if defined?(::ActiveJob)
 
-      all_queues = Shoryuken.queues
+      all_queues = Shoryuken.ungrouped_queues
       queues_with_workers = Shoryuken.worker_registry.queues
 
       (all_queues - queues_with_workers).each do |queue|
-        Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
+        Shoryuken.logger.warn { "No worker supplied for #{queue}" }
       end
     end
   end

data/lib/shoryuken/fetcher.rb

@@ -4,23 +4,23 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    def fetch(queue, available_processors)
+    attr_reader :group
+
+    def initialize(group)
+      @group = group
+    end
+
+    def fetch(queue, limit)
       started_at = Time.now
 
-      logger.debug { "Looking for new messages in '#{queue}'" }
+      logger.debug { "Looking for new messages in #{queue}" }
 
-      begin
-        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
+      sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
 
-        sqs_msgs = Array(receive_messages(queue, limit))
-        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
-        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
-        sqs_msgs
-      rescue => ex
-        logger.error { "Error fetching message: #{ex}" }
-        logger.error { ex.backtrace.first }
-        []
-      end
+      logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+      logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
+
+      sqs_msgs
     end
 
     private
@@ -29,10 +29,11 @@ module Shoryuken
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
-      options[:max_number_of_messages] = limit
+      options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
+
+      options[:max_number_of_messages]  = limit
       options[:message_attribute_names] = %w(All)
-      options[:attribute_names] = %w(All)
+      options[:attribute_names]         = %w(All)
 
       options.merge!(queue.options)
 

data/lib/shoryuken/launcher.rb

@@ -3,17 +3,96 @@ module Shoryuken
     include Util
 
     def initialize
-      @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
-                                        Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
+      @managers    = create_managers
+      @shutdowning = Concurrent::AtomicBoolean.new(false)
     end
 
-    def stop(options = {})
-      @manager.stop(shutdown: !options[:shutdown].nil?,
-                    timeout: Shoryuken.options[:timeout])
+    def start
+      logger.info { 'Starting' }
+
+      start_callback
+      start_managers
+    end
+
+    def stop!
+      initiate_stop
+
+      executor.shutdown
+
+      return if executor.wait_for_termination(Shoryuken.options[:timeout])
+
+      executor.kill
+    end
+
+    def stop
+      fire_event(:quiet, true)
+
+      initiate_stop
+
+      executor.shutdown
+      executor.wait_for_termination
+    end
+
+    private
+
+    def executor
+      Concurrent.global_io_executor
+    end
+
+    def start_managers
+      @managers.each do |manager|
+        Concurrent::Promise.execute { manager.start }.rescue do |ex|
+          log_manager_failure(ex)
+          start_soft_shutdown
+        end
+      end
+    end
+
+    def start_soft_shutdown
+      Process.kill('USR1', Process.pid) if @shutdowning.make_true
+    end
+
+    def log_manager_failure(ex)
+      return unless ex
+
+      logger.error { "Manager failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+    end
+
+    def initiate_stop
+      logger.info { 'Shutting down' }
+
+      @managers.each(&:stop)
+
+      stop_callback
+    end
+
+    def start_callback
+      if (callback = Shoryuken.start_callback)
+        logger.debug { 'Calling start_callback' }
+        callback.call
+      end
+
+      fire_event(:startup)
+    end
+
+    def stop_callback
+      if (callback = Shoryuken.stop_callback)
+        logger.debug { 'Calling stop_callback' }
+        callback.call
+      end
+
+      fire_event(:shutdown, true)
     end
 
-    def run
-      @manager.start
+    def create_managers
+      Shoryuken.groups.map do |group, options|
+        Shoryuken::Manager.new(
+          Shoryuken::Fetcher.new(group),
+          Shoryuken.polling_strategy(group).new(options[:queues]),
+          options[:concurrency]
+        )
+      end
     end
   end
 end