shoryuken 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9072ce45c134cd9563ee7dddac27bc7d19733cde
-  data.tar.gz: 2b48cb3b0d594ef1111456d0f9101ce154056995
+  metadata.gz: 08068a4a26c1f914715b25a27e463fac07e31b57
+  data.tar.gz: ac4fc670aac9fb76dec2c8c369f021ace729f0cd
 SHA512:
-  metadata.gz: 62bde57b564da861f90c2a68816a69743766f1120dd146817f884b2d4d2e5e5119dcef8858516b7d2581517a04177ec0edb151a97f254c6185ae9092c96efab2
-  data.tar.gz: e011b7d261eaf6e4b87d7dc322d2f458da09fdcd5074eac90eedb8101c082b2987f303b557ba20d90c89b321a7d4df7d613852890b6888263ccbc8adb1bf77db
+  metadata.gz: fdf1a335919582abb26bd34583de36da01b117b2e952b5d4874d40f4ee28b5d86cdca65fe9d4d52a807347569d2c9cb50bb20e82f8f6801ebf49aa2638219856
+  data.tar.gz: c9d462d292894b91ffe066af240ee5e7c6b8232542f19ba9d849a05bb9d7a0b6b732d6f96dbe6096610bbd9e4857f4540eed53bd5fd60b19c888dc8b88692e6b

data/.gitignore

@@ -24,3 +24,4 @@ shoryuken.yml
 *.pid
 *.log
 .env
+rubocop.html

data/.hound.yml

@@ -1,2 +1,6 @@
-StringLiterals:
-  EnforcedStyle: single_quotes
+ruby:
+  enabled: true
+  config_file: .rubocop.yml
+
+java_script:
+  enabled: true

data/.rubocop.yml

@@ -0,0 +1,50 @@
+LineLength:
+  Max: 120
+
+Style/SignalException:
+  Enabled: false
+
+Style/SpaceAroundEqualsInParameterDefault:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Style/CommentAnnotation:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Enabled: false
+
+Metrics/MethodLength :
+  Enabled: false
+
+Style/PerlBackrefs:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: single_quotes
+  SupportedStyles:
+    - single_quotes
+    - double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: single_quotes
+  SupportedStyles:
+    - single_quotes
+    - double_quotes
+
+Lint/UnusedMethodArgument:
+  Enabled: false

data/examples/bootstrap_queues.rb

@@ -22,7 +22,7 @@ if sqs.config['endpoint'] =~ /amazonaws.com/
   ).attributes['QueueArn']
 
   attributes = {}
-  attributes['RedrivePolicy'] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"}}
+  attributes['RedrivePolicy'] = %Q({"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"})
 
   sqs.set_queue_attributes queue_url: default_queue_url, attributes: attributes
 end

data/lib/shoryuken.rb

@@ -1,6 +1,5 @@
 require 'yaml'
 require 'aws-sdk-core'
-require 'aws-sdk-resources'
 require 'time'
 
 require 'shoryuken/version'
@@ -8,12 +7,15 @@ require 'shoryuken/core_ext'
 require 'shoryuken/util'
 require 'shoryuken/logging'
 require 'shoryuken/environment_loader'
+require 'shoryuken/queue'
+require 'shoryuken/message'
 require 'shoryuken/client'
 require 'shoryuken/worker'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
+require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
 require 'shoryuken/sns_arn'
 require 'shoryuken/topic'
@@ -76,6 +78,7 @@ module Shoryuken
         'delete'                  => false,
         'auto_delete'             => false,
         'auto_visibility_timeout' => false,
+        'retry_intervals'         => nil,
         'batch'                   => false }
     end
 
@@ -104,6 +107,7 @@ module Shoryuken
     def default_server_middleware
       Middleware::Chain.new do |m|
         m.add Middleware::Server::Timing
+        m.add Middleware::Server::ExponentialBackoffRetry
         m.add Middleware::Server::AutoDelete
         if defined?(::ActiveRecord::Base)
           require 'shoryuken/middleware/server/active_record'

data/lib/shoryuken/cli.rb

@@ -19,7 +19,7 @@ module Shoryuken
     def run(args)
       self_read, self_write = IO.pipe
 
-      %w[INT TERM USR1 USR2 TTIN].each do |sig|
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -48,7 +48,7 @@ module Shoryuken
       begin
         launcher.run
 
-        while readable_io = IO.select([self_read])
+        while (readable_io = IO.select([self_read]))
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
         end
@@ -86,7 +86,7 @@ module Shoryuken
 
       files_to_reopen.each do |file|
         begin
-          file.reopen file.path, "a+"
+          file.reopen file.path, 'a+'
           file.sync = true
         rescue ::Exception
         end
@@ -102,7 +102,7 @@ module Shoryuken
     end
 
     def write_pid
-      if path = Shoryuken.options[:pidfile]
+      if (path = Shoryuken.options[:pidfile])
         File.open(path, 'w') do |f|
           f.puts Process.pid
         end
@@ -150,7 +150,7 @@ module Shoryuken
           opts[:verbose] = arg
         end
 
-        o.on '-V', '--version', 'Print version and exit' do |arg|
+        o.on '-V', '--version', 'Print version and exit' do
           puts "Shoryuken #{Shoryuken::VERSION}"
           exit 0
         end
@@ -170,7 +170,7 @@ module Shoryuken
 
       case sig
       when 'USR1'
-        logger.info "Received USR1, will soft shutdown down"
+        logger.info 'Received USR1, will soft shutdown down'
 
         launcher.stop
 
@@ -181,7 +181,7 @@ module Shoryuken
           if thread.backtrace
             logger.info thread.backtrace.join("\n")
           else
-            logger.info "<no backtrace available>"
+            logger.info '<no backtrace available>'
           end
         end
 

data/lib/shoryuken/client.rb

@@ -5,7 +5,7 @@ module Shoryuken
 
     class << self
       def queues(name)
-        @@queues[name.to_s] ||= sqs_resource.get_queue_by_name(queue_name: name)
+        @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
       end
 
       def sns
@@ -20,10 +20,6 @@ module Shoryuken
         @sqs ||= Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
       end
 
-      def sqs_resource
-        @sqs_resource ||= Aws::SQS::Resource.new(client: sqs)
-      end
-
       def topics(name)
         @@topics[name.to_s] ||= Topic.new(name, sns)
       end
@@ -33,7 +29,7 @@ module Shoryuken
 
       private
 
-      def aws_client_options service_endpoint_key
+      def aws_client_options(service_endpoint_key)
         explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key]
         options = {}
         options[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?

data/lib/shoryuken/default_worker_registry.rb

@@ -28,7 +28,7 @@ module Shoryuken
     end
 
     def register_worker(queue, clazz)
-      if worker_class = @workers[queue]
+      if (worker_class = @workers[queue])
         if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
           raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
             "because #{worker_class} is already registered for this queue, "\

data/lib/shoryuken/environment_loader.rb

@@ -10,7 +10,7 @@ module Shoryuken
       load(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
     end
 
-    def initialize options
+    def initialize(options)
       @options = options
     end
 
@@ -64,8 +64,8 @@ module Shoryuken
 
       aws_options = aws_options.merge(credentials: credentials)
 
-      if callback = Shoryuken.aws_initialization_callback
-        Shoryuken.logger.info "Calling Shoryuken.on_aws_initialization block"
+      if (callback = Shoryuken.aws_initialization_callback)
+        Shoryuken.logger.info 'Calling Shoryuken.on_aws_initialization block'
         callback.call(aws_options)
       end
 
@@ -82,19 +82,19 @@ module Shoryuken
 
       require 'rails'
       if ::Rails::VERSION::MAJOR < 4
-        require File.expand_path("config/environment.rb")
+        require File.expand_path('config/environment.rb')
         ::Rails.application.eager_load!
       else
         # Painful contortions, see 1791 for discussion
-        require File.expand_path("config/application.rb")
-        ::Rails::Application.initializer "shoryuken.eager_load" do
+        require File.expand_path('config/application.rb')
+        ::Rails::Application.initializer 'shoryuken.eager_load' do
           ::Rails.application.config.eager_load = true
         end
         require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
-        require File.expand_path("config/environment.rb")
+        require File.expand_path('config/environment.rb')
       end
 
-      Shoryuken.logger.info "Rails environment loaded"
+      Shoryuken.logger.info 'Rails environment loaded'
     end
 
     def merge_cli_defined_queues

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -37,7 +37,6 @@ module ActiveJob
 
         def message(job, options = {})
           body = job.serialize
-          body = JSON.dump(body) if body.is_a?(Hash)
 
           { message_body: body,
             message_attributes: message_attributes }.merge(options)

data/lib/shoryuken/fetcher.rb

@@ -16,6 +16,7 @@ module Shoryuken
       options = Shoryuken.options[:aws][:receive_message].to_h.dup
       options[:max_number_of_messages] = limit
       options[:message_attribute_names] = %w(All)
+      options[:attribute_names] = %w(All)
 
       Shoryuken::Client.queues(queue).receive_messages options
     end
@@ -58,6 +59,7 @@ module Shoryuken
       end
 
     end
+
     private
 
     def patch_sqs_msgs!(sqs_msgs)

data/lib/shoryuken/logging.rb

@@ -17,12 +17,10 @@ module Shoryuken
     end
 
     def self.with_context(msg)
-      begin
-        Thread.current[:shoryuken_context] = msg
-        yield
-      ensure
-        Thread.current[:shoryuken_context] = nil
-      end
+      Thread.current[:shoryuken_context] = msg
+      yield
+    ensure
+      Thread.current[:shoryuken_context] = nil
     end
 
     def self.initialize_logger(log_target = STDOUT)

data/lib/shoryuken/manager.rb

@@ -32,8 +32,8 @@ module Shoryuken
       watchdog('Manager#stop died') do
         @done = true
 
-        if callback = Shoryuken.stop_callback
-          logger.info "Calling Shoryuken.on_stop block"
+        if (callback = Shoryuken.stop_callback)
+          logger.info 'Calling Shoryuken.on_stop block'
           callback.call
         end
 
@@ -89,7 +89,7 @@ module Shoryuken
     end
 
     def assign(queue, sqs_msg)
-      watchdog("Manager#assign died") do
+      watchdog('Manager#assign died') do
         logger.info "Assigning #{sqs_msg.message_id}"
 
         processor = @ready.pop
@@ -133,7 +133,7 @@ module Shoryuken
         return
       end
 
-      if queue = next_queue
+      if (queue = next_queue)
         @fetcher.async.fetch(queue, @ready.size)
       else
         logger.debug { 'Pausing fetcher, because all queues are paused' }
@@ -220,7 +220,7 @@ module Shoryuken
       logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
 
       after(delay) do
-        watchdog("Manager#hard_shutdown_in died") do
+        watchdog('Manager#hard_shutdown_in died') do
           if @busy.size > 0
             logger.info { "Hard shutting down #{@busy.size} busy workers" }
 

data/lib/shoryuken/message.rb

@@ -0,0 +1,60 @@
+module Shoryuken
+  class Message
+    attr_accessor :client, :queue_url, :data
+
+    def initialize(client, queue_url, data)
+      self.client = client
+      self.queue_url = queue_url
+      self.data = data
+    end
+
+    def delete
+      client.delete_message(
+        queue_url: queue_url,
+        receipt_handle: data.receipt_handle
+      )
+    end
+
+    def change_visibility(options)
+      client.change_message_visibility(
+        options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
+      )
+    end
+
+    def visibility_timeout=(timeout)
+      client.change_message_visibility(
+        queue_url: queue_url,
+        receipt_handle: data.receipt_handle,
+        visibility_timeout: timeout
+      )
+    end
+
+    def message_id
+      data.message_id
+    end
+
+    def receipt_handle
+      data.receipt_handle
+    end
+
+    def md5_of_body
+      data.md5_of_body
+    end
+
+    def body
+      data.body
+    end
+
+    def attributes
+      data.attributes
+    end
+
+    def md5_of_message_attributes
+      data.md5_of_message_attributes
+    end
+
+    def message_attributes
+      data.message_attributes
+    end
+  end
+end

data/lib/shoryuken/middleware/chain.rb

@@ -72,7 +72,7 @@ module Shoryuken
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
         i = entries.find_index { |entry| entry.klass == oldklass } || entries.count - 1
-        entries.insert(i+1, new_entry)
+        entries.insert(i + 1, new_entry)
       end
 
       def exists?(klass)

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -0,0 +1,50 @@
+module Shoryuken
+  module Middleware
+    module Server
+      class ExponentialBackoffRetry
+        include Util
+
+        def call(worker, queue, sqs_msg, body)
+          started_at = Time.now
+          yield
+        rescue
+          retry_intervals = Array(worker.class.get_shoryuken_options['retry_intervals'])
+
+          if retry_intervals.empty? || !handle_failure(sqs_msg, started_at, retry_intervals)
+            # Re-raise the exception if the job is not going to be exponential backoff retried.
+            # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
+            raise
+          end
+        end
+
+        private
+
+        def handle_failure(sqs_msg, started_at, retry_intervals)
+          attempts = sqs_msg.attributes['ApproximateReceiveCount']
+
+          return unless attempts
+
+          attempts = attempts.to_i - 1
+
+          interval = if attempts < retry_intervals.size
+                       retry_intervals[attempts]
+                     else
+                       retry_intervals.last
+                     end
+
+          # Visibility timeouts are limited to a total 12 hours, starting from the receipt of the message.
+          # We calculate the maximum timeout by subtracting the amount of time since the receipt of the message.
+          #
+          # From the docs:  "Amazon SQS restarts the timeout period using the new value."
+          # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/AboutVT.html#AboutVT-extending-message-visibility-timeout
+          max_timeout = 43200 - (Time.now - started_at).ceil - 1
+          interval = max_timeout if interval > max_timeout
+
+          sqs_msg.change_visibility(visibility_timeout: interval.to_i)
+
+          logger.info "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds."
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/queue.rb

@@ -0,0 +1,53 @@
+module Shoryuken
+  class Queue
+    attr_accessor :name, :client, :url
+
+    def initialize(client, name)
+      self.name   = name
+      self.client = client
+      self.url    = client.get_queue_url(queue_name: name).queue_url
+    end
+
+    def visibility_timeout
+      client.get_queue_attributes(
+        queue_url: url,
+        attribute_names: ['VisibilityTimeout']
+      ).attributes['VisibilityTimeout'].to_i
+    end
+
+    def delete_messages(options)
+      client.delete_message_batch(options.merge(queue_url: url))
+    end
+
+    def send_message(options)
+      client.send_message(sanitize_message_body(options.merge(queue_url: url)))
+    end
+
+    def send_messages(options)
+      client.send_message_batch(sanitize_message_body(options.merge(queue_url: url)))
+    end
+
+    def receive_messages(options)
+      client.receive_message(options.merge(queue_url: url)).
+        messages.
+        map { |m| Message.new(client, url, m) }
+    end
+
+    private
+
+    def sanitize_message_body(options)
+      messages = options[:entries] || [options]
+
+      messages.each do |m|
+        body = m[:message_body]
+        if body.is_a?(Hash)
+          m[:message_body] = JSON.dump(body)
+        elsif !body.is_a? String
+          fail ArgumentError, "The message body must be a String and you passed a #{body.class}"
+        end
+      end
+
+      options
+    end
+  end
+end

data/lib/shoryuken/sns_arn.rb

@@ -1,6 +1,6 @@
 module Shoryuken
   class SnsArn
-    def initialize topic
+    def initialize(topic)
       @topic = topic
     end
 
@@ -13,14 +13,14 @@ module Shoryuken
     def account_id
       Shoryuken::Client.account_id.tap do |account_id|
         if account_id.to_s.empty?
-          fail "To generate SNS ARNs, you must assign an :account_id in your Shoryuken::Client."
+          fail 'To generate SNS ARNs, you must assign an :account_id in your Shoryuken::Client.'
         end
       end
     end
 
     def region
       Aws.config.fetch(:region) do
-        fail "To generate SNS ARNs, you must include a :region in your AWS config."
+        fail 'To generate SNS ARNs, you must include a :region in your AWS config.'
       end
     end
   end

data/lib/shoryuken/util.rb

@@ -18,9 +18,8 @@ module Shoryuken
     end
 
     def unparse_queues(queues)
-      queues.inject({}) do |queue_and_weights, name|
+      queues.each_with_object({}) do |name, queue_and_weights|
         queue_and_weights[name] = queue_and_weights[name].to_i + 1
-        queue_and_weights
       end.to_a
     end
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/lib/shoryuken/worker.rb

@@ -13,7 +13,6 @@ module Shoryuken
           data_type: 'String'
         }
 
-        body = JSON.dump(body) if body.is_a?(Hash)
         options[:message_body] = body
 
         Shoryuken::Client.queues(get_shoryuken_options['queue']).send_message(options)

data/shoryuken.gemspec

@@ -4,27 +4,26 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'shoryuken/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "shoryuken"
+  spec.name          = 'shoryuken'
   spec.version       = Shoryuken::VERSION
-  spec.authors       = ["Pablo Cantero"]
-  spec.email         = ["pablo@pablocantero.com"]
-  spec.description = spec.summary = %q{Shoryuken is a super efficient AWS SQS thread based message processor}
-  spec.homepage      = "https://github.com/phstc/shoryuken"
-  spec.license       = "LGPL-3.0"
+  spec.authors       = ['Pablo Cantero']
+  spec.email         = ['pablo@pablocantero.com']
+  spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
+  spec.homepage      = 'https://github.com/phstc/shoryuken'
+  spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = %w[shoryuken]
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "nokogiri"
-  spec.add_development_dependency "dotenv"
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'nokogiri'
+  spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency "aws-sdk-core", "2.0.21"
-  spec.add_dependency "aws-sdk-resources", "2.0.21.pre"
-  spec.add_dependency "celluloid", "~> 0.16.0"
+  spec.add_dependency 'aws-sdk-core', '~> 2.0.21'
+  spec.add_dependency 'celluloid', '~> 0.16.0'
 end

data/spec/shoryuken/default_worker_registry_spec.rb

@@ -48,7 +48,7 @@ describe Shoryuken::DefaultWorkerRegistry do
         string_value: explicit_worker.to_s,
         data_type: 'String' } if explicit_worker
 
-      double Aws::SQS::Message,
+      double Shoryuken::Message,
         body: 'test',
         message_attributes: attributes,
         message_id: SecureRandom.uuid

data/spec/shoryuken/fetcher_spec.rb

@@ -4,11 +4,11 @@ require 'shoryuken/fetcher'
 
 describe Shoryuken::Fetcher do
   let(:manager)    { double Shoryuken::Manager }
-  let(:queue)      { double Aws::SQS::Queue }
+  let(:queue)      { double Shoryuken::Queue }
   let(:queue_name) { 'default' }
 
   let(:sqs_msg) do
-    double Aws::SQS::Message,
+    double Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
       message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
@@ -24,7 +24,7 @@ describe Shoryuken::Fetcher do
 
   describe '#fetch' do
     it 'calls pause when no message' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 1, message_attribute_names: ['All']).and_return([])
+      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).and_return([])
 
       expect(manager).to receive(:pause_queue!).with(queue_name)
       expect(manager).to receive(:dispatch)
@@ -33,7 +33,7 @@ describe Shoryuken::Fetcher do
     end
 
     it 'assigns messages' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, message_attribute_names: ['All']).and_return(sqs_msg)
+      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
 
       expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
       expect(manager).to receive(:assign).with(queue_name, sqs_msg)
@@ -45,7 +45,7 @@ describe Shoryuken::Fetcher do
     it 'assigns messages in batch' do
       TestWorker.get_shoryuken_options['batch'] = true
 
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: described_class::FETCH_LIMIT, message_attribute_names: ['All']).and_return(sqs_msg)
+      allow(queue).to receive(:receive_messages).with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
 
       expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
       expect(manager).to receive(:assign).with(queue_name, [sqs_msg])
@@ -58,7 +58,7 @@ describe Shoryuken::Fetcher do
       let(:queue_name) { 'notfound' }
 
       it 'ignores batch' do
-        allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, message_attribute_names: ['All']).and_return(sqs_msg)
+        allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
 
         expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
         expect(manager).to receive(:assign).with(queue_name, sqs_msg)

data/spec/shoryuken/middleware/server/auto_delete_spec.rb

@@ -2,10 +2,10 @@ require 'spec_helper'
 
 describe Shoryuken::Middleware::Server::AutoDelete do
   let(:queue)     { 'default' }
-  let(:sqs_queue) { double Aws::SQS::Queue }
+  let(:sqs_queue) { double Shoryuken::Queue }
 
   def build_message
-    double Aws::SQS::Message,
+    double Shoryuken::Message,
       queue_url: queue,
       body: 'test',
       receipt_handle: SecureRandom.uuid

data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
+  let(:queue)     { 'default' }
+  let(:sqs_queue) { double Shoryuken::Queue }
+  let(:sqs_msg)   { double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid,
+                    attributes: {'ApproximateReceiveCount' => 1}, message_id: SecureRandom.uuid }
+
+  before do
+    allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+  end
+
+  context 'when a job succeeds' do
+    it 'does not retry the job' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
+
+      expect(sqs_msg).not_to receive(:change_visibility)
+
+      subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) {}
+    end
+  end
+
+  context 'when a job throws an exception' do
+
+    it 'does not retry the job by default' do
+      expect(sqs_msg).not_to receive(:change_visibility)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.to raise_error
+    end
+
+    it 'does not retry the job if :retry_intervals is empty' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = []
+
+      expect(sqs_msg).not_to receive(:change_visibility)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.to raise_error
+    end
+
+    it 'retries the job if :retry_intervals is non-empty' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
+
+      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+    end
+
+    it 'retries the job with exponential backoff' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
+
+      allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 2 } }
+      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+    end
+
+    it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
+
+      allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 3 } }
+      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+    end
+
+    it 'limits the visibility timeout to 12 hours from receipt of message' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [86400]
+
+      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
+
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+    end
+  end
+end

data/spec/shoryuken/middleware/server/timing_spec.rb

@@ -2,10 +2,10 @@ require 'spec_helper'
 
 describe Shoryuken::Middleware::Server::Timing do
   let(:queue) { 'default' }
-  let(:sqs_queue) { double Aws::SQS::Queue, visibility_timeout: 60 }
+  let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 60 }
 
   let(:sqs_msg) do
-    double Aws::SQS::Message,
+    double Shoryuken::Message,
       queue_url: queue,
       body: 'test',
       message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'

data/spec/shoryuken/processor_spec.rb

@@ -4,11 +4,11 @@ require 'shoryuken/manager'
 
 describe Shoryuken::Processor do
   let(:manager)   { double Shoryuken::Manager, processor_done: nil }
-  let(:sqs_queue) { double Aws::SQS::Queue, visibility_timeout: 30 }
+  let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
 
   let(:sqs_msg) do
-    double Aws::SQS::Message,
+    double Shoryuken::Message,
       queue_url: queue,
       body: 'test',
       message_attributes: {},
@@ -38,7 +38,7 @@ describe Shoryuken::Processor do
     end
 
     it 'parses the body calling the proc' do
-      TestWorker.get_shoryuken_options['body_parser'] = Proc.new { |sqs_msg| "*#{sqs_msg.body}*" }
+      TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
 
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
 
@@ -207,15 +207,15 @@ describe Shoryuken::Processor do
 
     context 'when shoryuken_class header' do
       let(:sqs_msg) do
-        double Aws::SQS::Message,
+        double Shoryuken::Message,
           queue_url: queue,
           body: 'test',
           message_attributes: {
             'shoryuken_class' => {
               string_value: TestWorker.to_s,
               data_type: 'String' }},
-          message_id: SecureRandom.uuid,
-          receipt_handle: SecureRandom.uuid
+              message_id: SecureRandom.uuid,
+              receipt_handle: SecureRandom.uuid
       end
 
       it 'performs without delete' do
@@ -244,7 +244,7 @@ describe Shoryuken::Processor do
 
     context 'when the worker takes a long time', slow: true do
       it 'extends the message invisibility to prevent it from being dequeued concurrently' do
-        TestWorker.get_shoryuken_options['body_parser'] = Proc.new do |sqs_msg|
+        TestWorker.get_shoryuken_options['body_parser'] = proc do
           sleep visibility_timeout
           'test'
         end

data/spec/shoryuken/queue_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Shoryuken::Queue do
+  let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
+  let(:sqs)         { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
+  let(:queue_name)  { 'shoryuken' }
+  let(:queue_url)   { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
+
+  subject { described_class.new(sqs, queue_name) }
+
+  describe '#send_message' do
+    context 'when body is invalid' do
+      it 'raises ArgumentError for nil' do
+        expect {
+          subject.send_message(message_body: nil)
+        }.to raise_error(ArgumentError, 'The message body must be a String and you passed a NilClass')
+      end
+
+      it 'raises ArgumentError for Fixnum' do
+        expect {
+          subject.send_message(message_body: 1)
+        }.to raise_error(ArgumentError, 'The message body must be a String and you passed a Fixnum')
+      end
+    end
+  end
+
+  describe '#send_messages' do
+    context 'when body is invalid' do
+      it 'raises ArgumentError for nil' do
+        expect {
+          subject.send_messages(entries: [message_body: nil])
+        }.to raise_error(ArgumentError, 'The message body must be a String and you passed a NilClass')
+      end
+
+      it 'raises ArgumentError for Fixnum' do
+        expect {
+          subject.send_messages(entries: [message_body: 1])
+        }.to raise_error(ArgumentError, 'The message body must be a String and you passed a Fixnum')
+      end
+    end
+  end
+end

data/spec/shoryuken/util_spec.rb

@@ -17,7 +17,7 @@ describe 'Shoryuken::Util' do
 
   describe '#worker_name' do
     let(:sqs_msg) do
-      double Aws::SQS::Message, message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', message_attributes: {}
+      double Shoryuken::Message, message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', message_attributes: {}
     end
 
     it 'returns Shoryuken worker name' do

data/spec/shoryuken/worker_spec.rb

@@ -66,19 +66,6 @@ describe 'Shoryuken::Worker' do
       TestWorker.perform_async('message')
     end
 
-    it 'enqueues a message given as hash' do
-      expect(sqs_queue).to receive(:send_message).with(
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: '{"field":"part1","other_field":"part2"}')
-
-      TestWorker.perform_async(field: 'part1', other_field: 'part2')
-    end
-
     it 'enqueues a message with options' do
       expect(sqs_queue).to receive(:send_message).with(
         delay_seconds: 60,

data/spec/spec_helper.rb

@@ -42,9 +42,6 @@ RSpec.configure do |config|
   config.filter_run_excluding slow: true unless ENV['SPEC_ALL']
 
   config.before do
-    # remove doubles, preventing:
-    # Double "Queue" was originally created in one example but has leaked into another example and can no longer be used.
-    # rspec-mocks' doubles are designed to only last for one example, and you need to create a new one in each example you wish to use it for.
     Shoryuken::Client.class_variable_set :@@queues, {}
     Shoryuken::Client.class_variable_set :@@visibility_timeouts, {}
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-14 00:00:00.000000000 Z
+date: 2015-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -98,30 +98,16 @@ dependencies:
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.0.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.0.21
-- !ruby/object:Gem::Dependency
-  name: aws-sdk-resources
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.0.21.pre
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.0.21.pre
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
@@ -147,6 +133,7 @@ files:
 - .gitignore
 - .hound.yml
 - .rspec
+- .rubocop.yml
 - .travis.yml
 - Gemfile
 - LICENSE.txt
@@ -166,11 +153,14 @@ files:
 - lib/shoryuken/launcher.rb
 - lib/shoryuken/logging.rb
 - lib/shoryuken/manager.rb
+- lib/shoryuken/message.rb
 - lib/shoryuken/middleware/chain.rb
 - lib/shoryuken/middleware/server/active_record.rb
 - lib/shoryuken/middleware/server/auto_delete.rb
+- lib/shoryuken/middleware/server/exponential_backoff_retry.rb
 - lib/shoryuken/middleware/server/timing.rb
 - lib/shoryuken/processor.rb
+- lib/shoryuken/queue.rb
 - lib/shoryuken/sns_arn.rb
 - lib/shoryuken/topic.rb
 - lib/shoryuken/util.rb
@@ -188,8 +178,10 @@ files:
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb
+- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb
+- spec/shoryuken/queue_spec.rb
 - spec/shoryuken/sns_arn_spec.rb
 - spec/shoryuken/topic_spec.rb
 - spec/shoryuken/util_spec.rb
@@ -230,8 +222,10 @@ test_files:
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb
+- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb
+- spec/shoryuken/queue_spec.rb
 - spec/shoryuken/sns_arn_spec.rb
 - spec/shoryuken/topic_spec.rb
 - spec/shoryuken/util_spec.rb