shoryuken 2.0.4 → 2.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25d7b8d6df3ab9ede1f0d7a6aba01d18e2a102f2
-  data.tar.gz: bf26e631ee57665f7b18d0a314259b48e71951cd
+  metadata.gz: 9859d893c7542fbdac0224f68e55a8b49f0fd984
+  data.tar.gz: d3e5419d74c6766c95c4b8ad20333bdced3cfd57
 SHA512:
-  metadata.gz: edfb890c3b31a38252bfecfd5f0f1c4d98b31c892f12106136fd91c973434f544e76c1ca48c860f2cd4f1065342231ca1dbd3a3f8752eb49613be45b57ce4c87
-  data.tar.gz: 3c36a03cb7bc41ce4038c5942b87f3e5b087d08a4e26e9387f9dc3222bbd4e50840af7ddd68f3b073712de5d8a629d57d29ec9387e855c16499183983fea0e04
+  metadata.gz: cb5725e4def20f4c82a785db1c8a055ba59892979608ce95855888bfbd321c5310f491284a2d6ef319e7bc1b3c1f186d91e376cc6b35faef600c26d4fa453325
+  data.tar.gz: 15274a15cd3dc7f067d4ae54a40921b90b703e09e2002ed32049ce41f6dd1261629de3b3fa842656d574455f01b13e4558536997c93d4f5ebd05880f1c80b9e2

data/CHANGELOG.md

@@ -1,4 +1,55 @@
-## [v2.0.4] -
+## [v2.0.10] - 2016-06-09
+
+- Fix manager #225
+ - [#226] https://github.com/phstc/shoryuken/pull/226
+
+## [v2.0.9] - 2016-06-08
+
+- Fix daemonization broken in #219
+ - [#224] https://github.com/phstc/shoryuken/pull/224
+
+## [v2.0.8] - 2016-06-07
+
+- Fix daemonization
+ - [#223] https://github.com/phstc/shoryuken/pull/223
+
+## [v2.0.7] - 2016-06-06
+
+- Daemonize before loading environment
+ - [#219] https://github.com/phstc/shoryuken/pull/219
+
+- Fix initialization when using rails
+ - [#197] https://github.com/phstc/shoryuken/pull/197
+
+- Improve message fetching
+ - https://github.com/phstc/shoryuken/pull/214 and https://github.com/phstc/shoryuken/commit/f4640d97950c1783a061195855d93994725ed64a
+
+- Fix hard shutdown if there are some busy workers when signal received
+ - [#215] https://github.com/phstc/shoryuken/pull/215
+
+- Fix `rake console` task
+ - [#208] https://github.com/phstc/shoryuken/pull/208
+
+- Isolate `MessageVisibilityExtender` as new middleware
+ - [#199] https://github.com/phstc/shoryuken/pull/190
+
+- Fail on non-existent queues
+ - [#196] https://github.com/phstc/shoryuken/pull/196
+
+## [v2.0.6] - 2016-04-18
+
+- Fix log initialization introduced by #191
+ - [#195](https://github.com/phstc/shoryuken/pull/195)
+
+## [v2.0.5] - 2016-04-17
+
+- Fix log initialization when using `Shoryuken::EnvironmentLoader#load`
+ - [#191](https://github.com/phstc/shoryuken/pull/191)
+
+ - Fix `enqueue_at` in the ActiveJob Adapter
+ - [#182](https://github.com/phstc/shoryuken/pull/182)
+
+## [v2.0.4] - 2016-02-04
 
 - Add Rails 3 support
  - [#175](https://github.com/phstc/shoryuken/pull/175)

data/README.md

@@ -105,7 +105,7 @@ end
 
 [Check the Middleware documentation](https://github.com/phstc/shoryuken/wiki/Middleware).
 
-### Configuration
+### Configuration (worker side)
 
 Sample configuration file `shoryuken.yml`.
 
@@ -136,6 +136,19 @@ The ```aws``` section is used to configure both the Aws objects used by Shoryuke
 
 The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
 
+### Configuration (producer side)
+
+'Producer' processes need permissions to put messages into SQS. There are a few ways:
+
+* Ensure the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars are set.
+* Create a `~/.aws/credentials` file.
+* Set `Aws.config[:credentials]` from Ruby code (e.g. in a Rails initializer)
+* Use the Instance Profiles feature. The IAM role of the targeted machine must have an adequate SQS Policy.
+
+Note that storing your credentials into Amazon instances represents a security risk. Instance Profiles tends to be the best choice.
+
+You can read about these in more detail [here](http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html).
+
 ### Rails Integration
 
 [Check the Rails Integration Active Job documention](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job).

data/Rakefile

@@ -1,9 +1,16 @@
 require 'bundler/gem_tasks'
 $stdout.sync = true
 
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
 desc 'Open Shoryuken pry console'
 task :console do
   require 'pry'
+  require 'celluloid'
   require 'shoryuken'
 
   config_file = File.join File.expand_path('..', __FILE__), 'shoryuken.yml'

data/lib/shoryuken.rb

@@ -15,6 +15,7 @@ require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
+Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
 require 'shoryuken/sns_arn'
@@ -126,8 +127,8 @@ module Shoryuken
     #     end
     #   end
     def on(event, &block)
-      raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-      raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
       options[:lifecycle_events][event] << block
     end
 
@@ -142,6 +143,7 @@ module Shoryuken
         m.add Middleware::Server::Timing
         m.add Middleware::Server::ExponentialBackoffRetry
         m.add Middleware::Server::AutoDelete
+        m.add Middleware::Server::AutoExtendVisibility
         if defined?(::ActiveRecord::Base)
           require 'shoryuken/middleware/server/active_record'
           m.add Middleware::Server::ActiveRecord

data/lib/shoryuken/cli.rb

@@ -31,10 +31,11 @@ module Shoryuken
 
       options = parse_cli_args(args)
 
+      daemonize(options)
+      write_pid(options)
+
       EnvironmentLoader.load(options)
 
-      daemonize
-      write_pid
       load_celluloid
 
       require 'shoryuken/launcher'
@@ -63,21 +64,27 @@ module Shoryuken
     private
 
     def load_celluloid
-      raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization" if defined?(::Celluloid) && Shoryuken.options[:daemon]
-
-      # Celluloid can't be loaded until after we've daemonized
-      # because it spins up threads and creates locks which get
-      # into a very bad state if forked.
       require 'celluloid/autostart'
       Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
 
       require 'shoryuken/manager'
     end
 
-    def daemonize
-      return unless Shoryuken.options[:daemon]
+    def celluloid_loaded?
+      defined?(::Celluloid)
+    end
+
+    def daemonize(options)
+      return unless options[:daemon]
+
+      fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
 
-      raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless Shoryuken.options[:logfile]
+      if celluloid_loaded?
+        # Celluloid can't be loaded until after we've daemonized
+        # because it spins up threads and creates locks which get
+        # into a very bad state if forked.
+        raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization"
+      end
 
       files_to_reopen = []
       ObjectSpace.each_object(File) do |file|
@@ -95,7 +102,7 @@ module Shoryuken
       end
 
       [$stdout, $stderr].each do |io|
-        File.open(Shoryuken.options[:logfile], 'ab') do |f|
+        File.open(options[:logfile], 'ab') do |f|
           io.reopen(f)
         end
         io.sync = true
@@ -103,8 +110,8 @@ module Shoryuken
       $stdin.reopen('/dev/null')
     end
 
-    def write_pid
-      if (path = Shoryuken.options[:pidfile])
+    def write_pid(options)
+      if (path = options[:pidfile])
         File.open(path, 'w') do |f|
           f.puts Process.pid
         end
@@ -112,7 +119,7 @@ module Shoryuken
     end
 
     def parse_cli_args(argv)
-      opts = { queues: [] }
+      opts = {}
 
       @parser = OptionParser.new do |o|
         o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
@@ -125,6 +132,7 @@ module Shoryuken
 
         o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
           queue, weight = arg.split(',')
+          opts[:queues] = [] unless opts[:queues]
           opts[:queues] << [queue, weight]
         end
 

data/lib/shoryuken/default_worker_registry.rb

@@ -30,9 +30,9 @@ module Shoryuken
     def register_worker(queue, clazz)
       if (worker_class = @workers[queue])
         if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+          fail ArgumentError, "Could not register #{clazz} for '#{queue}', "\
             "because #{worker_class} is already registered for this queue, "\
-            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers."
+            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
         end
       end
 

data/lib/shoryuken/environment_loader.rb

@@ -15,12 +15,11 @@ module Shoryuken
     end
 
     def load
-      initialize_logger
       load_rails if options[:rails]
-      Shoryuken.options.merge!(config_file_options)
+      initialize_options
+      initialize_logger
       merge_cli_defined_queues
       prefix_active_job_queue_names
-      Shoryuken.options.merge!(options)
       parse_queues
       require_workers
       initialize_aws
@@ -31,15 +30,15 @@ module Shoryuken
 
     private
 
+    def initialize_options
+      Shoryuken.options.merge!(config_file_options)
+      Shoryuken.options.merge!(options)
+    end
+
     def config_file_options
-      if (path = options[:config_file])
-        unless File.exist?(path)
-          Shoryuken.logger.warn { "Config file #{path} does not exist" }
-          path = nil
-        end
-      end
+      return {} unless (path = options[:config_file])
 
-      return {} unless path
+      fail ArgumentError, "The supplied config file '#{path}' does not exist" unless File.exist?(path)
 
       YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
     end
@@ -96,8 +95,6 @@ module Shoryuken
         require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
         require File.expand_path('config/environment.rb')
       end
-
-      Shoryuken.logger.info { 'Rails environment loaded' }
     end
 
     def merge_cli_defined_queues
@@ -162,13 +159,17 @@ module Shoryuken
     def validate_queues
       Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
 
+      non_existent_queues = []
+
       Shoryuken.queues.uniq.each do |queue|
         begin
           Shoryuken::Client.queues queue
         rescue Aws::SQS::Errors::NonExistentQueue
-          Shoryuken.logger.warn { "The specified queue '#{queue}' does not exist" }
+          non_existent_queues << queue
         end
       end
+
+      fail ArgumentError, "The specified queue(s) #{non_existent_queues} do not exist" if non_existent_queues.any?
     end
 
     def validate_workers

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -26,7 +26,7 @@ module ActiveJob
         end
 
         def enqueue_at(job, timestamp)
-          instance.enqueue(job, timestamp)
+          instance.enqueue_at(job, timestamp)
         end
       end
 

data/lib/shoryuken/fetcher.rb

@@ -47,15 +47,13 @@ module Shoryuken
             @manager.async.pause_queue!(queue)
           end
 
-          @manager.async.dispatch
-
           logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
         rescue => ex
           logger.error { "Error fetching message: #{ex}" }
           logger.error { ex.backtrace.first }
-
-          @manager.async.dispatch
         end
+
+        @manager.async.dispatch
       end
 
     end

data/lib/shoryuken/manager.rb

@@ -11,7 +11,8 @@ module Shoryuken
     trap_exit :processor_died
 
     def initialize(condvar)
-      @count  = Shoryuken.options[:concurrency] || 25
+      @count = Shoryuken.options[:concurrency] || 25
+      raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
       @queues = Shoryuken.queues.dup.uniq
       @finished = condvar
 
@@ -67,6 +68,7 @@ module Shoryuken
 
         if stopped?
           processor.terminate if processor.alive?
+          return after(0) { @finished.signal } if @busy.empty?
         else
           @ready << processor
         end
@@ -80,7 +82,9 @@ module Shoryuken
         @threads.delete(processor.object_id)
         @busy.delete processor
 
-        unless stopped?
+        if stopped?
+          return after(0) { @finished.signal } if @busy.empty?
+        else
           @ready << build_processor
         end
       end

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -0,0 +1,53 @@
+require 'celluloid' unless defined?(Celluloid)
+
+module Shoryuken
+  module Middleware
+    module Server
+      class AutoExtendVisibility
+        EXTEND_UPFRONT_SECONDS = 5
+
+        def call(worker, queue, sqs_msg, body)
+          timer = auto_visibility_timer(queue, sqs_msg, worker.class)
+          begin
+            yield
+          ensure
+            timer.cancel if timer
+          end
+        end
+
+        private
+
+        class MessageVisibilityExtender
+          include Celluloid
+          include Util
+
+          def auto_extend(queue, sqs_msg, worker_class)
+            queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
+
+            every(queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+              begin
+                logger.debug do
+                  "Extending message #{worker_name(worker_class, sqs_msg)}/#{queue}/#{sqs_msg.message_id}  " \
+                               "visibility timeout by #{queue_visibility_timeout}s."
+                end
+
+                sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
+              rescue => e
+                logger.error do
+                  "Could not auto extend the message #{worker_class}/#{queue}/#{sqs_msg.message_id} " \
+                               "visibility timeout. Error: #{e.message}"
+                end
+              end
+            end
+          end
+        end
+
+        def auto_visibility_timer(queue, sqs_msg, worker_class)
+          return unless worker_class.auto_visibility_timeout?
+          @visibility_extender ||= MessageVisibilityExtender.new_link
+          @visibility_extender.auto_extend(queue, sqs_msg, worker_class)
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/processor.rb

@@ -16,9 +16,6 @@ module Shoryuken
 
       worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
 
-      timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
-
-      begin
         body = get_body(worker.class, sqs_msg)
 
         worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
@@ -26,40 +23,10 @@ module Shoryuken
         end
 
         @manager.async.processor_done(queue, current_actor)
-      ensure
-        timer.cancel if timer
-      end
     end
 
     private
 
-    class MessageVisibilityExtender
-      include Celluloid
-      include Util
-
-      def auto_extend(queue, sqs_msg, worker_class)
-        queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
-
-        every(queue_visibility_timeout - 5) do
-          begin
-            logger.debug { "Extending message #{worker_name(worker_class, sqs_msg)}/#{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s." }
-
-            sqs_msg.visibility_timeout = queue_visibility_timeout
-          rescue => e
-            logger.error { "Could not auto extend the message #{worker_class}/#{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{e.message}" }
-          end
-        end
-      end
-    end
-
-    def auto_visibility_timeout(queue, sqs_msg, worker_class)
-      return unless worker_class.auto_visibility_timeout?
-
-      @visibility_extender ||= MessageVisibilityExtender.new_link
-
-      @visibility_extender.auto_extend(queue, sqs_msg, worker_class)
-    end
-
     def get_body(worker_class, sqs_msg)
       if sqs_msg.is_a? Array
         sqs_msg.map { |m| parse_body(worker_class, m) }

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '2.0.4'
+  VERSION = '2.0.11'
 end

data/shoryuken.gemspec

@@ -6,8 +6,8 @@ require 'shoryuken/version'
 Gem::Specification.new do |spec|
   spec.name          = 'shoryuken'
   spec.version       = Shoryuken::VERSION
-  spec.authors       = ['Pablo Cantero']
-  spec.email         = ['pablo@pablocantero.com']
+  spec.authors       = ['Pablo Cantero', 'Mario Kostelac']
+  spec.email         = ['pablo@pablocantero.com', 'mariokostelac@gmail.com']
   spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
   spec.homepage      = 'https://github.com/phstc/shoryuken'
   spec.license       = 'LGPL-3.0'

data/spec/shoryuken/cli_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+require 'shoryuken/cli'
+require 'shoryuken/launcher'
+
+RSpec.describe Shoryuken::CLI do
+  let(:cli) { Shoryuken::CLI.instance }
+
+  before(:each) do
+    # make sure we do not bail
+    allow(cli).to receive(:exit)
+
+    # make sure we do not mess with standard streams
+    allow_any_instance_of(IO).to receive(:reopen)
+  end
+
+  describe '#run' do
+    let(:launcher) { instance_double('Shoryuken::Launcher') }
+
+    before(:each) do
+      allow(Shoryuken::Launcher).to receive(:new).and_return(launcher)
+      allow(launcher).to receive(:run).and_raise(Interrupt)
+      allow(launcher).to receive(:stop)
+    end
+
+    it 'does not raise' do
+      expect { cli.run([]) }.to_not raise_error
+    end
+
+    it 'daemonizes with --daemon --logfile' do
+      expect(cli).to receive(:celluloid_loaded?).and_return(false)
+      expect(Process).to receive(:daemon)
+      cli.run(['--daemon', '--logfile', '/dev/null'])
+    end
+
+    it 'does NOT daemonize with --daemon --logfile' do
+      expect(Process).to_not receive(:daemon)
+      cli.run(['--logfile', '/dev/null'])
+    end
+
+    it 'writes PID file with --pidfile' do
+      pidfile = instance_double('File')
+      expect(File).to receive(:open).with('/dev/null', 'w').and_yield(pidfile)
+      expect(pidfile).to receive(:puts).with(Process.pid)
+      cli.run(['--pidfile', '/dev/null'])
+    end
+  end
+
+  describe '#daemonize' do
+    before(:each) do
+      allow(cli).to receive(:celluloid_loaded?).and_return(false)
+    end
+
+    it 'raises if logfile is not set' do
+      expect { cli.send(:daemonize, daemon: true) }.to raise_error(ArgumentError)
+    end
+
+    it 'raises if Celluloid is already loaded' do
+      expect(cli).to receive(:celluloid_loaded?).and_return(true)
+      args = { daemon: true, logfile: '/dev/null' }
+      expect { cli.send(:daemonize, args) }.to raise_error(RuntimeError)
+    end
+
+    it 'calls Process.daemon' do
+      args = { daemon: true, logfile: '/dev/null' }
+      expect(Process).to receive(:daemon).with(true, true)
+      cli.send(:daemonize, args)
+    end
+  end
+end

data/spec/shoryuken/core_ext_spec.rb

@@ -2,20 +2,20 @@ require 'spec_helper'
 
 RSpec.describe 'Core Extensions' do
   describe Hash do
-    describe 'stringify_keys' do
+    describe '#stringify_keys' do
       it 'converts keys into strings' do
         expect({ :key1 => 'value1', 'key2' => 'value2' }.stringify_keys).to eq('key1' => 'value1', 'key2' => 'value2')
       end
     end
 
-    describe 'symbolize_keys' do
-      it 'converts keys into strings' do
+    describe '#symbolize_keys' do
+      it 'converts keys into symbols' do
         expect({ :key1 => 'value1', 'key2' => 'value2' }.symbolize_keys).to eq(:key1 => 'value1', key2: 'value2')
       end
     end
 
-    describe 'deep_symbolize_keys' do
-      it 'converts keys into strings' do
+    describe '#deep_symbolize_keys' do
+      it 'converts keys into symbols' do
         expect({ :key1 => 'value1',
                  'key2' => 'value2',
                  'key3' => {

data/spec/shoryuken/manager_spec.rb

@@ -1,13 +1,22 @@
 require 'spec_helper'
 require 'shoryuken/manager'
 
-describe Shoryuken::Manager do
+RSpec.describe Shoryuken::Manager do
   subject do
     condvar = double(:condvar)
     allow(condvar).to receive(:signal).and_return(nil)
     Shoryuken::Manager.new(condvar)
   end
 
+  describe 'Invalid concurrency setting' do
+    it 'raises ArgumentError if concurrency is not positive number' do
+      Shoryuken.options[:concurrency] = -1
+      expect { Shoryuken::Manager.new(nil) }
+        .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
+    end
+
+  end
+
   describe 'Auto Scaling' do
     it 'decreases weight' do
       queue1 = 'shoryuken'

data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe Shoryuken::Middleware::Server::AutoExtendVisibility do
+  let(:queue) { 'default' }
+  let(:visibility_timeout) { 3 }
+  let(:extend_upfront) { 1 }
+  let(:sqs_queue) { instance_double Shoryuken::Queue, visibility_timeout: visibility_timeout }
+
+  def build_message
+    double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid
+  end
+
+  # We need to run our worker inside actor context.
+  class Runner
+    include Celluloid
+
+    def run_and_sleep(worker, queue, sqs_msg, interval)
+      Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
+        sleep interval
+      end
+    end
+
+    def run_and_raise(worker, queue, sqs_msg, error_class)
+      Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
+        raise error_class.new
+      end
+    end
+  end
+
+  let(:sqs_msg) { build_message }
+
+  before do
+    allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+    stub_const('Shoryuken::Middleware::Server::AutoExtendVisibility::EXTEND_UPFRONT_SECONDS', extend_upfront)
+  end
+
+  it 'extends message visibility if jobs takes a long time' do
+    TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
+
+    allow(sqs_msg).to receive(:queue) { sqs_queue }
+    expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: visibility_timeout)
+
+    Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout)
+  end
+
+  it 'does not extend message visibility if worker raises' do
+    TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
+
+    allow(sqs_msg).to receive(:queue) { sqs_queue }
+    expect(sqs_msg).to_not receive(:change_visibility)
+
+    expect { Runner.new.run_and_raise(TestWorker.new, queue, sqs_msg, StandardError) }.to raise_error(StandardError)
+  end
+
+  it 'does not extend message visibility if auto_visibility_timeout is not true' do
+    TestWorker.get_shoryuken_options['auto_visibility_timeout'] = false
+
+    allow(sqs_msg).to receive(:queue) { sqs_queue }
+    expect(sqs_msg).to_not receive(:change_visibility)
+
+    Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout)
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 require 'shoryuken/processor'
 require 'shoryuken/manager'
 
-describe Shoryuken::Processor do
+RSpec.describe Shoryuken::Processor do
   let(:manager)   { double Shoryuken::Manager, processor_done: nil }
   let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
@@ -100,14 +100,16 @@ describe Shoryuken::Processor do
 
       it 'logs the error' do
         expect(subject.logger).to receive(:error) do |&block|
-          expect(block.call).to eq("Error parsing the message body: 757: unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
+          expect(block.call).
+            to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
         end
 
         subject.process(queue, sqs_msg) rescue nil
       end
 
       it 're raises the error' do
-        expect{ subject.process(queue, sqs_msg) }.to raise_error(JSON::ParserError, "757: unexpected token at 'invalid json'")
+        expect { subject.process(queue, sqs_msg) }.
+          to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
       end
     end
 
@@ -257,50 +259,4 @@ describe Shoryuken::Processor do
       end
     end
   end
-
-  describe '#auto_visibility_timeout' do
-    let(:heartbeat)          { sqs_queue.visibility_timeout - 5 }
-    let(:visibility_timeout) { sqs_queue.visibility_timeout }
-
-    before do
-      TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
-
-      allow(sqs_queue).to receive(:visibility_timeout).and_return(15)
-    end
-
-    context 'when the worker takes a long time', slow: true do
-      it 'extends the message invisibility to prevent it from being dequeued concurrently' do
-        TestWorker.get_shoryuken_options['body_parser'] = proc do
-          sleep visibility_timeout
-          'test'
-        end
-
-        expect(sqs_msg).to receive(:visibility_timeout=).with(visibility_timeout).once
-        expect(manager).to receive(:processor_done).with(queue, subject)
-
-        allow(sqs_msg).to receive(:body).and_return('test')
-
-        subject.process(queue, sqs_msg)
-      end
-    end
-
-    context 'when the worker takes a short time' do
-      it 'does not extend the message invisibility' do
-        expect(sqs_msg).to receive(:visibility_timeout=).never
-        expect(manager).to receive(:processor_done).with(queue, subject)
-
-        allow(sqs_msg).to receive(:body).and_return('test')
-
-        subject.process(queue, sqs_msg)
-      end
-    end
-
-    context 'when the worker fails' do
-      it 'does not extend the message invisibility' do
-        expect(sqs_msg).to receive(:visibility_timeout=).never
-        expect_any_instance_of(TestWorker).to receive(:perform).and_raise 'worker failed'
-        expect { subject.process(queue, sqs_msg) }.to raise_error(RuntimeError, 'worker failed')
-      end
-    end
-  end
 end

data/spec/shoryuken_spec.rb

@@ -45,7 +45,7 @@ describe Shoryuken do
           def perform(sqs_msg, body); end
         end
       }.to raise_error("Could not register BatchableWorker for 'default', because TestWorker is already registered for this queue, " \
-                       "and Shoryuken doesn't support a batchable worker for a queue with multiple workers.")
+                       "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
     end
   end
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.11
 platform: ruby
 authors:
 - Pablo Cantero
+- Mario Kostelac
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-14 00:00:00.000000000 Z
+date: 2016-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -125,6 +126,7 @@ dependencies:
 description: Shoryuken is a super efficient AWS SQS thread based message processor
 email:
 - pablo@pablocantero.com
+- mariokostelac@gmail.com
 executables:
 - shoryuken
 extensions: []
@@ -158,6 +160,7 @@ files:
 - lib/shoryuken/middleware/chain.rb
 - lib/shoryuken/middleware/server/active_record.rb
 - lib/shoryuken/middleware/server/auto_delete.rb
+- lib/shoryuken/middleware/server/auto_extend_visibility.rb
 - lib/shoryuken/middleware/server/exponential_backoff_retry.rb
 - lib/shoryuken/middleware/server/timing.rb
 - lib/shoryuken/processor.rb
@@ -172,6 +175,7 @@ files:
 - shoryuken.jpg
 - spec/integration/launcher_spec.rb
 - spec/shoryuken.yml
+- spec/shoryuken/cli_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
@@ -179,6 +183,7 @@ files:
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb
+- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb
@@ -210,13 +215,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Shoryuken is a super efficient AWS SQS thread based message processor
 test_files:
 - spec/integration/launcher_spec.rb
 - spec/shoryuken.yml
+- spec/shoryuken/cli_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
@@ -224,6 +230,7 @@ test_files:
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb
+- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb