shoryuken 3.0.4 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1367b27e316c258563ef328b78bc64947c849d8
-  data.tar.gz: 285db9748d737bfe07a6904db381d3c11883cd05
+  metadata.gz: a7a48b27440a94923275b030e8785feb20e3b628
+  data.tar.gz: 629c1e8369fef39bf795860d3cdd565f2ed6c886
 SHA512:
-  metadata.gz: 7f85154f339098c0b04a01673548b0c688217290772131dc28a1ed0c378b0cb671576f5a195143d915a8ab70ed836b089bd12ff6c4d6aa78d81bf858729323a6
-  data.tar.gz: 9b5180a9baf239178c11afcc7f55d31392162deeb95aa7f49836a64e94fcc114b145ba82414b83ad22d278612d3cca53436bb804d41d98ccfbe9f1eea4f678cc
+  metadata.gz: 77e1c8fb04eea615ae5ba684978c5278631a3dab89119d9519ee3e0514da3ca3cdeb12feb672beb2024f5f708d986953b1b6a0191ea4c3e75e3e507809c710e3
+  data.tar.gz: dbff1f35986729de23d4934dccab52546642297334f1117c925c89fdbf8735575ae2dd0bc990e547018a60921a19f59cd04968093d73072609156ac8ff4541f5

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [v3.0.5] - 2017-04-09
+- Pause endless dispatcher to avoid CPU overload
+ - [#354](https://github.com/phstc/shoryuken/pull/354)
+
+- Auto log processor errors
+ - [#355](https://github.com/phstc/shoryuken/pull/355)
+
+- Add a delay as a CLI param
+ - [#350](https://github.com/phstc/shoryuken/pull/350)
+
+- Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
+ - [#344](https://github.com/phstc/shoryuken/pull/344)
 ## [v3.0.4] - 2017-03-24
 - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
  - [#344](https://github.com/phstc/shoryuken/pull/344)

data/bin/shoryuken

@@ -21,14 +21,22 @@ module Shoryuken
       method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
       method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
       method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
-      method_option :config_file, aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
       method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
       method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
       method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
       method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
+      method_option :delay,       aliases: '-D', type: :boolean, desc: 'Number of seconds to pause fetching from an empty queue'
       def start
         opts = options.to_h.symbolize_keys
 
+        if opts[:config_file]
+          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+        end
+
+        opts[:config_file] = opts.delete(:config) if opts[:config]
+
         # Keep compatibility with old CLI queue format
         opts[:queues] = options[:queues].map { |q| q.split(',') } if options[:queues]
 

data/lib/shoryuken/manager.rb

@@ -3,6 +3,8 @@ module Shoryuken
     include Util
 
     BATCH_LIMIT = 10
+    # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
+    MIN_DISPATCH_INTERVAL = 0.1
 
     def initialize(fetcher, polling_strategy)
       @count = Shoryuken.options.fetch(:concurrency, 25)
@@ -47,6 +49,11 @@ module Shoryuken
       end
     end
 
+    def processor_failed(ex)
+      logger.error ex
+      logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+    end
+
     def processor_done(queue)
       logger.debug { "Process done for '#{queue}'" }
     end
@@ -61,8 +68,10 @@ module Shoryuken
       return if @done.true?
 
       begin
-        return if ready.zero?
-        return unless (queue = @polling_strategy.next_queue)
+        if ready.zero? || (queue = @polling_strategy.next_queue).nil?
+          sleep MIN_DISPATCH_INTERVAL
+          return
+        end
 
         logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 

data/lib/shoryuken/processor.rb

@@ -13,6 +13,9 @@ module Shoryuken
       worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
         worker.perform(sqs_msg, body)
       end
+    rescue Exception => ex
+      @manager.processor_failed(ex)
+      raise
     ensure
       @manager.processor_done(queue)
     end

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.0.4'.freeze
+  VERSION = '3.0.5'.freeze
 end

data/spec/shoryuken/processor_spec.rb

@@ -99,6 +99,7 @@ RSpec.describe Shoryuken::Processor do
       end
 
       it 'logs the error' do
+        expect(manager).to receive(:processor_failed)
         expect(subject.logger).to receive(:error) do |&block|
           expect(block.call).
             to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
@@ -108,6 +109,7 @@ RSpec.describe Shoryuken::Processor do
       end
 
       it 're raises the error' do
+        expect(manager).to receive(:processor_failed)
         expect { subject.process(queue, sqs_msg) }.
           to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-24 00:00:00.000000000 Z
+date: 2017-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler