shoryuken 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3afa1e42164a3fc885caf45222d317ac77692c3b
-  data.tar.gz: 9abfa8e2c9ed9b4540bbb422fba35a3f00591364
+  metadata.gz: 70a39ae890a6fe9dcff44b0c6ceeab9c77931ab2
+  data.tar.gz: ff223f629ed2f60cc0abb32fa17f28e15d680d2f
 SHA512:
-  metadata.gz: 35934726404b3d5454b1bc97875d3ba8934d2ef4efa4a354dbff97af95959c647757259ffc6fefc53a8ffaab5c1cfeffad73d79aff50a3575f5e65d39cbbb553
-  data.tar.gz: 6cafc6fbfd06fff62541acc41169877440454eba4afb1688728bbd69330ed53fa9a62d1c28b2b9a057de853952cff26d537d013747a8cc5b6c47a8b637f3c1f2
+  metadata.gz: f0afdb29904c40200d1976f2b26973ae6694e7018b07bf133153c355e250a1d613cf1fd9c0500c2aaa612d77f4c5d631823ab186642b4a2051a76ea78feea4e6
+  data.tar.gz: 8b3d0dc3f255b3d921386a322e645e7875f2962c902094a48671d24a95211b02e5f0558be53e30619baf92551e9f3d7e23b60def11a2c11b84107525e339f141

data/README.md

@@ -30,13 +30,12 @@ If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but
 
 If `high_priority` receives a new message, Shoryuken will smoothly increase back the `high_priority` weight one by one until it reaches the weight of 6 again, which is the maximum configured for `high_priority`.
 
-If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again.
+If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again. [Check the delay option documentation for more information](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
 
-*You can set `delay: 0` to continuously check the queues without pausing even if they are empty.*
 
 ### Fetch in batches
 
-To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches.
+To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
 
 ## Installation
 
@@ -44,7 +43,7 @@ Add this line to your application's Gemfile:
 
     gem 'shoryuken'
 
-**Require Shoryuken from GitHub to get the latest updates:**
+Or to get the latest updates:
 
     gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
 
@@ -64,7 +63,7 @@ Or install it yourself as:
 class MyWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'default', delete: true
+  shoryuken_options queue: 'default', auto_delete: true
   # shoryuken_options queue: ->{ "#{ENV['environment']_default" }
 
   # shoryuken_options body_parser: :json
@@ -81,16 +80,7 @@ end
 
 ### Sending a message
 
-```ruby
-MyWorker.perform_async('Pablo')
-# or
-Shoryuken::Client.queues('default').send_message('Pablo')
-
-# delaying a message
-MyWorker.perform_async('Pablo', delay_seconds: 60)
-# or
-Shoryuken::Client.queues('default').send_message('Pablo', delay_seconds: 60)
-```
+[Check the Sending a message documentation](https://github.com/phstc/shoryuken/wiki/Sending-a-message)
 
 ### Midleware
 
@@ -128,6 +118,14 @@ queues:
   - [low_priority, 1]
 ```
 
+### Rails Integration
+
+You can tell Shoryuken to load your Rails application by passing the `-R` or `--rails` flag to the "shoryuken" command.
+
+If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
+
+This feature works for Rails 4+, but needs to be confirmed for older versions.
+
 ### Start Shoryuken
 
 ```shell
@@ -145,6 +143,7 @@ shoryuken [options]
     -q, --queue QUEUE[,WEIGHT]...    Queues to process with optional weights
     -r, --require [PATH|DIR]         Location of the worker
     -C, --config PATH                Path to YAML config file
+    -R, --rails                      Attempts to load the containing Rails project
     -L, --logfile PATH               Path to writable logfile
     -P, --pidfile PATH               Path to pidfile
     -v, --verbose                    Print more verbose output

data/Rakefile

@@ -21,27 +21,3 @@ task :console do
   ARGV.clear
   Pry.start
 end
-
-desc 'Push test messages to high_priority, default and low_priority'
-task :push_test, :size do |t, args|
-  require 'yaml'
-  require 'shoryuken'
-
-  config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
-
-  AWS.config(config['aws'])
-
-  Shoryuken::Client.sqs.queues.create('default')
-  Shoryuken::Client.sqs.queues.create('high_priority')
-  Shoryuken::Client.sqs.queues.create('low_priority')
-
-  (args[:size] || 1).to_i.times.map do |i|
-    Thread.new do
-      puts "Pushing test ##{i}"
-
-      Shoryuken::Client.queues('high_priority').send_message("test #{i}")
-      Shoryuken::Client.queues('default').send_message("test #{i}")
-      Shoryuken::Client.queues('low_priority').send_message("test #{i}")
-    end
-  end.each &:join
-end

data/examples/bootstrap_queues.rb

@@ -0,0 +1,20 @@
+require 'yaml'
+require 'shoryuken'
+
+# load SQS credentials
+config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
+
+AWS.config(config['aws'])
+
+sqs = AWS::SQS.new
+
+# create a queue and a respective dead letter queue
+# after 7 attempts SQS will move the message to the dead letter queue
+
+dl_name = 'default_failures'
+dl = sqs.queues.create(dl_name)
+
+options = {}
+options[:redrive_policy] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dl.arn}"}"}
+
+sqs.queues.create('default', options)

data/examples/default_worker.rb

@@ -1,9 +1,20 @@
 class DefaultWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'default', delete: true, body_parser: ->(sqs_msg){ "new body: #{sqs_msg.body}" }
+  shoryuken_options queue: 'default', auto_delete: true
 
   def perform(sqs_msg, body)
     puts "DefaultWorker: '#{body}'"
   end
 end
+
+# multiple workers for the same queue
+class DefaultWorker2
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'default', auto_delete: true
+
+  def perform(sqs_msg, body)
+    puts "DefaultWorker2: '#{body}'"
+  end
+end

data/lib/shoryuken.rb

@@ -1,5 +1,5 @@
 require 'yaml'
-require 'aws-sdk'
+require 'aws-sdk-v1'
 require 'time'
 
 require 'shoryuken/version'
@@ -7,6 +7,7 @@ require 'shoryuken/core_ext'
 require 'shoryuken/util'
 require 'shoryuken/client'
 require 'shoryuken/worker'
+require 'shoryuken/worker_loader'
 require 'shoryuken/logging'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
@@ -21,52 +22,75 @@ module Shoryuken
     timeout: 8
   }
 
-  @@workers = {}
-  @@queues = []
+  @@workers       = {}
+  @@queues        = []
+  @@worker_loader = WorkerLoader
 
-  def self.options
-    @options ||= DEFAULTS.dup
-  end
+  class << self
+    def options
+      @options ||= DEFAULTS.dup
+    end
 
-  def self.register_worker(queue, clazz)
-    @@workers[queue] = clazz
-  end
+    def register_worker(queue, clazz)
+      if worker_class = @@workers[queue]
+        if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
+          raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+            "because #{worker_class} is already registered for this queue, "\
+            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
+        end
+      end
 
-  def self.workers
-    @@workers
-  end
+      @@workers[queue] = clazz
+    end
 
-  def self.queues
-    @@queues
-  end
+    def workers
+      @@workers
+    end
 
-  def self.logger
-    Shoryuken::Logging.logger
-  end
+    def queues
+      @@queues
+    end
 
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #   end
-  # end
-  def self.configure_server
-    yield self
-  end
+    def logger
+      Shoryuken::Logging.logger
+    end
 
-  def self.server_middleware
-    @server_chain ||= default_server_middleware
-    yield @server_chain if block_given?
-    @server_chain
-  end
+    def worker_loader=(worker_loader)
+      @@worker_loader = worker_loader
+    end
+
+    def worker_loader
+      @@worker_loader
+    end
+
+    # Shoryuken.configure_server do |config|
+    #   config.server_middleware do |chain|
+    #     chain.add MyServerHook
+    #   end
+    # end
+    def configure_server
+      yield self
+    end
+
+    def server_middleware
+      @server_chain ||= default_server_middleware
+      yield @server_chain if block_given?
+      @server_chain
+    end
 
 
-  private
+    private
 
-  def self.default_server_middleware
-    Middleware::Chain.new do |m|
-      m.add Middleware::Server::Timing
-      m.add Middleware::Server::AutoDelete
-      # TODO m.add Middleware::Server::RetryJobs
+    def default_server_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Server::Timing
+        m.add Middleware::Server::AutoDelete
+        if defined?(::ActiveRecord::Base)
+          require 'shoryuken/middleware/server/active_record'
+          m.add Middleware::Server::ActiveRecord
+        end
+        # TODO m.add Middleware::Server::RetryJobs
+      end
     end
   end
 end

data/lib/shoryuken/cli.rb

@@ -22,7 +22,10 @@ module Shoryuken
         end
       end
 
-      setup_options(args)
+      setup_options(args) do |cli_options|
+        # this needs to happen before configuration is parsed, since it may depend on Rails env
+        load_rails if cli_options[:rails]
+      end
       initialize_logger
       require_workers
       validate!
@@ -61,6 +64,25 @@ module Shoryuken
       require 'shoryuken/manager'
     end
 
+    def load_rails
+      # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
+
+      require 'rails'
+      if ::Rails::VERSION::MAJOR < 4
+        require File.expand_path("config/environment.rb")
+        ::Rails.application.eager_load!
+      else
+        # Painful contortions, see 1791 for discussion
+        require File.expand_path("config/application.rb")
+        ::Rails::Application.initializer "shoryuken.eager_load" do
+          ::Rails.application.config.eager_load = true
+        end
+        require File.expand_path("config/environment.rb")
+      end
+
+      logger.info "Rails environment loaded"
+    end
+
     def daemonize
       return unless Shoryuken.options[:daemon]
 
@@ -125,6 +147,10 @@ module Shoryuken
           opts[:config_file] = arg
         end
 
+        o.on '-R', '--rails', 'Load Rails' do |arg|
+          opts[:rails] = arg
+        end
+
         o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
           opts[:logfile] = arg
         end
@@ -187,6 +213,9 @@ module Shoryuken
     def setup_options(args)
       options = parse_options(args)
 
+      # yield parsed options in case we need to do more setup before configuration is parsed
+      yield(options) if block_given?
+
       config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
 
       Shoryuken.options.merge!(config)
@@ -213,8 +242,8 @@ module Shoryuken
     def validate!
       raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
 
-      if queue_without_worker = Shoryuken.queues.find { |queue| Shoryuken.workers[queue].nil? }
-        raise ArgumentError, "No worker supplied for #{queue_without_worker}"
+      Shoryuken.queues.each do |queue|
+        logger.warn "No worker supplied for '#{queue}'" unless Shoryuken.workers.include? queue
       end
 
       if Shoryuken.options[:aws][:access_key_id].nil? && Shoryuken.options[:aws][:secret_access_key].nil?

data/lib/shoryuken/client.rb

@@ -17,6 +17,8 @@ module Shoryuken
       end
 
       def send_message(queue, body, options = {})
+        body = JSON.dump(body) if body.is_a?(Hash)
+
         queues(queue).send_message(body, options)
       end
 

data/lib/shoryuken/fetcher.rb

@@ -13,17 +13,22 @@ module Shoryuken
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
+      options = Shoryuken.options[:aws][:receive_message].to_h
+      options[:limit] = limit
+      options[:message_attribute_names] ||= []
+      options[:message_attribute_names] << 'shoryuken_class'
+
+      Shoryuken::Client.receive_message queue, options
     end
 
     def fetch(queue, available_processors)
       watchdog('Fetcher#fetch died') do
         started_at = Time.now
 
-        logger.info "Looking for new messages '#{queue}'"
+        logger.debug "Looking for new messages in '#{queue}'"
 
         begin
-          batch = !!Shoryuken.workers[queue].get_shoryuken_options['batch']
+          batch = !!(Shoryuken.workers[queue] && Shoryuken.workers[queue].get_shoryuken_options['batch'])
 
           limit = batch ? FETCH_LIMIT : available_processors
 
@@ -38,7 +43,7 @@ module Shoryuken
 
             @manager.async.rebalance_queue_weight!(queue)
           else
-            logger.info "No message found for '#{queue}'"
+            logger.debug "No message found for '#{queue}'"
 
             @manager.async.pause_queue!(queue)
           end

data/lib/shoryuken/manager.rb

@@ -103,7 +103,7 @@ module Shoryuken
     def pause_queue!(queue)
       return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
 
-      logger.info "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
+      logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
 
       @queues.delete(queue)
 
@@ -139,7 +139,7 @@ module Shoryuken
       return if stopped?
 
       unless @queues.include? queue
-        logger.info "Restarting '#{queue}'"
+        logger.debug "Restarting '#{queue}'"
 
         @queues << queue
 
@@ -169,7 +169,21 @@ module Shoryuken
     def next_queue
       return nil if @queues.empty?
 
+      # get/remove the first queue in the list
       queue = @queues.shift
+
+
+      unless Shoryuken.workers.include? queue
+        # when no worker registered pause the queue to avoid endless recursion
+
+        logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because of no workers registered"
+
+        after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
+
+        return next_queue
+      end
+
+      # add queue back to the end of the list
       @queues << queue
 
       queue

data/lib/shoryuken/middleware/server/active_record.rb

@@ -0,0 +1,13 @@
+module Shoryuken
+  module Middleware
+    module Server
+      class ActiveRecord
+        def call(*args)
+          yield
+        ensure
+          ::ActiveRecord::Base.clear_active_connections!
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -2,14 +2,12 @@ module Shoryuken
   module Middleware
     module Server
       class AutoDelete
-
         def call(worker, queue, sqs_msg, body)
           yield
 
-          # I'm still deciding, but `auto_delete` will be probably deprecated soon
-          delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
+          auto_delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
 
-          Shoryuken::Client.queues(queue).batch_delete(*Array(sqs_msg)) if delete
+          Shoryuken::Client.queues(queue).batch_delete(*Array(sqs_msg)) if auto_delete
         end
       end
     end

data/lib/shoryuken/processor.rb

@@ -10,11 +10,10 @@ module Shoryuken
     end
 
     def process(queue, sqs_msg)
-      worker_class = Shoryuken.workers[queue]
-      defer do
-        body = get_body(worker_class, sqs_msg)
+      worker = Shoryuken.worker_loader.call(queue, sqs_msg)
 
-        worker = worker_class.new
+      defer do
+        body = get_body(worker.class, sqs_msg)
 
         Shoryuken.server_middleware.invoke(worker, queue, sqs_msg, body) do
           worker.perform(sqs_msg, body)

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/shoryuken/worker.rb

@@ -6,19 +6,43 @@ module Shoryuken
 
     module ClassMethods
       def perform_async(body, options = {})
+        options ||= {}
+        options[:message_attributes] ||= {}
+        options[:message_attributes]['shoryuken_class'] = {
+          string_value: self.to_s,
+          data_type: 'String'
+        }
+
         Shoryuken::Client.send_message(get_shoryuken_options['queue'], body, options)
       end
 
+      def perform_in(interval, body, options = {})
+        interval = interval.to_f
+        now = Time.now.to_f
+        ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
+
+        delay = (ts - now).ceil
+
+        raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
+
+        perform_async(body, options.merge(delay_seconds: delay))
+      end
+
+      alias_method :perform_at, :perform_in
+
       def shoryuken_options(opts = {})
         @shoryuken_options = get_shoryuken_options.merge(stringify_keys(Hash(opts)))
         queue = @shoryuken_options['queue']
-        queue = queue.call if queue.respond_to? :call
+        if queue.respond_to? :call
+          queue = queue.call
+          @shoryuken_options['queue'] = queue
+        end
 
         Shoryuken.register_worker(queue, self)
       end
 
       def get_shoryuken_options # :nodoc:
-        @shoryuken_options || { 'queue' => 'default', 'delete' => false, 'batch' => false }
+        @shoryuken_options || { 'queue' => 'default', 'delete' => false, 'auto_delete' => false, 'batch' => false }
       end
 
       def stringify_keys(hash) # :nodoc:

data/lib/shoryuken/worker_loader.rb

@@ -0,0 +1,17 @@
+module Shoryuken
+  class WorkerLoader
+    class << self
+      def call(queue, sqs_msg)
+        # missing `try?` - yes, I'm
+        worker_class = !sqs_msg.is_a?(Array) &&
+          sqs_msg.message_attributes &&
+          sqs_msg.message_attributes['shoryuken_class'] &&
+          sqs_msg.message_attributes['shoryuken_class'][:string_value]
+
+        worker_class = (worker_class.constantize rescue nil) || Shoryuken.workers[queue]
+
+        worker_class.new
+      end
+    end
+  end
+end

data/shoryuken.gemspec

@@ -22,6 +22,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry-byebug"
 
-  spec.add_dependency "aws-sdk"
-  spec.add_dependency "celluloid"
+  spec.add_dependency "aws-sdk-v1"
+  spec.add_dependency "celluloid", "~> 0.15.2"
 end

data/spec/integration/launcher_spec.rb

@@ -9,19 +9,45 @@ describe Shoryuken::Launcher do
       Shoryuken.options[:aws][:receive_message] ||= {}
       Shoryuken.options[:aws][:receive_message][:wait_time_seconds] = 5
 
+      Shoryuken.queues << 'shoryuken'
+      Shoryuken.queues << 'shoryuken_command'
+
+      Shoryuken.register_worker 'shoryuken',          StandardWorker
+      Shoryuken.register_worker 'shoryuken_command',  CommandWorker
+
       subject.run
 
-      ShoryukenWorker.received_messages = 0
+      StandardWorker.received_messages = 0
     end
 
     after { subject.stop }
 
-    class ShoryukenWorker
+    class CommandWorker
+      include Shoryuken::Worker
+
+      @@received_messages = 0
+
+      shoryuken_options queue: 'shoryuken_command', auto_delete: true
+
+      def perform(sqs_msg, body)
+        @@received_messages = Array(sqs_msg).size
+      end
+
+      def self.received_messages
+        @@received_messages
+      end
+
+      def self.received_messages=(received_messages)
+        @@received_messages = received_messages
+      end
+    end
+
+    class StandardWorker
       include Shoryuken::Worker
 
       @@received_messages = 0
 
-      shoryuken_options queue: 'shoryuken', delete: true
+      shoryuken_options queue: 'shoryuken', auto_delete: true
 
       def perform(sqs_msg, body)
         @@received_messages = Array(sqs_msg).size
@@ -36,31 +62,42 @@ describe Shoryuken::Launcher do
       end
     end
 
+    it 'consumes as a command worker' do
+      CommandWorker.perform_async('Yo')
+
+      10.times do
+        break if CommandWorker.received_messages > 0
+        sleep 1
+      end
+
+      expect(CommandWorker.received_messages).to eq 1
+    end
+
     it 'consumes a message' do
-      ShoryukenWorker.get_shoryuken_options['batch'] = false
+      StandardWorker.get_shoryuken_options['batch'] = false
 
       Shoryuken::Client.queues('shoryuken').send_message('Yo')
 
       10.times do
-        break if ShoryukenWorker.received_messages > 0
-        sleep 0.2
+        break if StandardWorker.received_messages > 0
+        sleep 1
       end
 
-      expect(ShoryukenWorker.received_messages).to eq 1
+      expect(StandardWorker.received_messages).to eq 1
     end
 
     it 'consumes a batch' do
-      ShoryukenWorker.get_shoryuken_options['batch'] = true
+      StandardWorker.get_shoryuken_options['batch'] = true
 
       Shoryuken::Client.queues('shoryuken').batch_send *(['Yo'] * 10)
 
       10.times do
-        break if ShoryukenWorker.received_messages > 0
-        sleep 0.2
+        break if StandardWorker.received_messages > 0
+        sleep 1
       end
 
       # the fetch result is uncertain, should be greater than 1, but hard to tell the exact size
-      expect(ShoryukenWorker.received_messages).to be > 1
+      expect(StandardWorker.received_messages).to be > 1
     end
   end
 end

data/spec/shoryuken/client_spec.rb

@@ -33,6 +33,22 @@ describe Shoryuken::Client do
 
       described_class.send_message(queue, 'test2', delay_seconds: 60)
     end
+
+    it 'parsers as JSON by default' do
+      msg = { field: 'test', other_field: 'other' }
+
+      expect(sqs_queue).to receive(:send_message).with(JSON.dump(msg), {})
+
+      described_class.send_message(queue, msg)
+    end
+
+    it 'parsers as JSON by default and keep the options' do
+      msg = { field: 'test', other_field: 'other' }
+
+      expect(sqs_queue).to receive(:send_message).with(JSON.dump(msg), { delay_seconds:  60 })
+
+      described_class.send_message(queue, msg, delay_seconds: 60)
+    end
   end
 
   describe '.visibility_timeout' do

data/spec/shoryuken/fetcher_spec.rb

@@ -18,7 +18,7 @@ describe Shoryuken::Fetcher do
 
   describe '#fetch' do
     it 'calls pause when no message' do
-      allow(sqs_queue).to receive(:receive_message).with(limit: 1).and_return([])
+      allow(sqs_queue).to receive(:receive_message).with(limit: 1, message_attribute_names: ['shoryuken_class']).and_return([])
 
       expect(manager).to receive(:pause_queue!).with(queue)
       expect(manager).to receive(:dispatch)
@@ -27,7 +27,7 @@ describe Shoryuken::Fetcher do
     end
 
     it 'assigns messages' do
-      allow(sqs_queue).to receive(:receive_message).with(limit: 5).and_return(sqs_msg)
+      allow(sqs_queue).to receive(:receive_message).with(limit: 5, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
 
       expect(manager).to receive(:rebalance_queue_weight!).with(queue)
       expect(manager).to receive(:assign).with(queue, sqs_msg)
@@ -39,7 +39,7 @@ describe Shoryuken::Fetcher do
     it 'assigns messages in batch' do
       TestWorker.get_shoryuken_options['batch'] = true
 
-      allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT).and_return(sqs_msg)
+      allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
 
       expect(manager).to receive(:rebalance_queue_weight!).with(queue)
       expect(manager).to receive(:assign).with(queue, [sqs_msg])
@@ -47,5 +47,19 @@ describe Shoryuken::Fetcher do
 
       subject.fetch(queue, 5)
     end
+
+    context 'when worker not found' do
+      let(:queue) { 'notfound' }
+
+      it 'ignores batch' do
+        allow(sqs_queue).to receive(:receive_message).with(limit: 5, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
+
+        expect(manager).to receive(:rebalance_queue_weight!).with(queue)
+        expect(manager).to receive(:assign).with(queue, sqs_msg)
+        expect(manager).to receive(:dispatch)
+
+        subject.fetch(queue, 5)
+      end
+    end
   end
 end

data/spec/shoryuken/manager_spec.rb

@@ -71,4 +71,37 @@ describe Shoryuken::Manager do
       expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
     end
   end
+
+  describe '#next_queue' do
+    it 'returns queues' do
+      queue1 = 'shoryuken'
+      queue2 = 'uppercut'
+
+      Shoryuken.queues.clear
+
+      Shoryuken.register_worker queue1, TestWorker
+      Shoryuken.register_worker queue2, TestWorker
+
+      Shoryuken.queues << queue1
+      Shoryuken.queues << queue2
+
+      expect(subject.send :next_queue).to eq queue1
+      expect(subject.send :next_queue).to eq queue2
+    end
+
+    it 'skips when no worker' do
+      queue1 = 'shoryuken'
+      queue2 = 'uppercut'
+
+      Shoryuken.queues.clear
+
+      Shoryuken.register_worker queue2, TestWorker
+
+      Shoryuken.queues << queue1
+      Shoryuken.queues << queue2
+
+      expect(subject.send :next_queue).to eq queue2
+      expect(subject.send :next_queue).to eq queue2
+    end
+  end
 end

data/spec/shoryuken/middleware/server/auto_delete_spec.rb

@@ -10,7 +10,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
   end
 
   it 'deletes a message' do
-    TestWorker.get_shoryuken_options['delete'] = true
+    TestWorker.get_shoryuken_options['auto_delete'] = true
 
     expect(sqs_queue).to receive(:batch_delete).with(sqs_msg)
 
@@ -18,7 +18,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
   end
 
   it 'deletes a batch' do
-    TestWorker.get_shoryuken_options['delete'] = true
+    TestWorker.get_shoryuken_options['auto_delete'] = true
 
     sqs_msg2 = double 'SQS msg', body: 'test'
     sqs_msg3 = double 'SQS msg', body: 'test'
@@ -31,7 +31,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
   end
 
   it 'does not delete a message' do
-    TestWorker.get_shoryuken_options['delete'] = false
+    TestWorker.get_shoryuken_options['auto_delete'] = false
 
     expect(sqs_queue).to_not receive(:batch_delete)
 

data/spec/shoryuken/processor_spec.rb

@@ -6,7 +6,7 @@ describe Shoryuken::Processor do
   let(:manager)   { double Shoryuken::Manager, processor_done: nil }
   let(:sqs_queue) { double AWS::SQS::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
-  let(:sqs_msg)   { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test' }
+  let(:sqs_msg)   { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {} }
 
   subject { described_class.new(manager) }
 
@@ -16,7 +16,7 @@ describe Shoryuken::Processor do
   end
 
   describe '#process' do
-    it 'parses the body into JSON' do
+    it 'parsers the body into JSON' do
       TestWorker.get_shoryuken_options['body_parser'] = :json
 
       body = { 'test' => 'hi' }
@@ -28,7 +28,7 @@ describe Shoryuken::Processor do
       subject.process(queue, sqs_msg)
     end
 
-    it 'parses the body calling the proc' do
+    it 'parsers the body calling the proc' do
       TestWorker.get_shoryuken_options['body_parser'] = Proc.new { |sqs_msg| "*#{sqs_msg.body}*" }
 
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
@@ -38,7 +38,7 @@ describe Shoryuken::Processor do
       subject.process(queue, sqs_msg)
     end
 
-    it 'parses the body as text' do
+    it 'parsers the body as text' do
       TestWorker.get_shoryuken_options['body_parser'] = :text
 
       body = 'test'
@@ -50,7 +50,7 @@ describe Shoryuken::Processor do
       subject.process(queue, sqs_msg)
     end
 
-    it 'parses calling `.parse`' do
+    it 'parsers calling `.parse`' do
       TestWorker.get_shoryuken_options['body_parser'] = JSON
 
       body = { 'test' => 'hi' }
@@ -77,7 +77,7 @@ describe Shoryuken::Processor do
     end
 
     context 'when `object_type: nil`' do
-      it 'parses the body as text' do
+      it 'parsers the body as text' do
         TestWorker.get_shoryuken_options['body_parser'] = nil
 
         body = 'test'
@@ -126,7 +126,7 @@ describe Shoryuken::Processor do
     end
 
     it 'performs with delete' do
-      TestWorker.get_shoryuken_options['delete'] = true
+      TestWorker.get_shoryuken_options['auto_delete'] = true
 
       expect(manager).to receive(:processor_done).with(queue, subject)
 
@@ -138,7 +138,7 @@ describe Shoryuken::Processor do
     end
 
     it 'performs without delete' do
-      TestWorker.get_shoryuken_options['delete'] = false
+      TestWorker.get_shoryuken_options['auto_delete'] = false
 
       expect(manager).to receive(:processor_done).with(queue, subject)
 
@@ -148,5 +148,26 @@ describe Shoryuken::Processor do
 
       subject.process(queue, sqs_msg)
     end
+
+    context 'when shoryuken_class header' do
+      let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {
+        'shoryuken_class' => {
+          string_value: TestWorker.to_s,
+          data_type: 'String'
+        }
+      } }
+
+      it 'performs without delete' do
+        Shoryuken.workers.clear # unregister TestWorker
+
+        expect(manager).to receive(:processor_done).with(queue, subject)
+
+        expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
+
+        expect(sqs_queue).to_not receive(:batch_delete)
+
+        subject.process(queue, sqs_msg)
+      end
+    end
   end
 end

data/spec/shoryuken/worker_loader_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Shoryuken::WorkerLoader do
+  let(:queue)   { 'default' }
+  let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: { } }
+
+  describe '.call' do
+    it 'returns the worker using `Shoryuken.workers`' do
+      expect(described_class.call(queue, sqs_msg)).to be_an_instance_of TestWorker
+    end
+
+    context 'when `message_attributes`' do
+      let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {
+        'shoryuken_class' => {
+          string_value: TestWorker.to_s,
+          data_type: 'String'
+        }
+      } }
+
+      it 'returns the worker using `message_attributes`' do
+        Shoryuken.workers.clear
+
+        expect(described_class.call(queue, sqs_msg)).to be_an_instance_of TestWorker
+      end
+    end
+  end
+end

data/spec/shoryuken/worker_spec.rb

@@ -8,15 +8,74 @@ describe 'Shoryuken::Worker' do
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
+  describe '.perform_in' do
+    it 'delays a message' do
+      expect(sqs_queue).to receive(:send_message).with('message', {
+        message_attributes: {
+          'shoryuken_class' => {
+            string_value: TestWorker.to_s,
+            data_type: 'String'
+          }
+        },
+        delay_seconds: 60
+      })
+
+      TestWorker.perform_in(60, 'message')
+    end
+
+    it 'raises an exception' do
+      expect {
+        TestWorker.perform_in(901, 'message')
+      }.to raise_error 'The maximum allowed delay is 15 minutes'
+    end
+  end
+
+  describe '.perform_at' do
+    it 'delays a message' do
+      expect(sqs_queue).to receive(:send_message).with('message', {
+        message_attributes: {
+          'shoryuken_class' => {
+            string_value: TestWorker.to_s,
+            data_type: 'String'
+          }
+        },
+        delay_seconds: 60
+      })
+
+      TestWorker.perform_in(Time.now + 60, 'message')
+    end
+
+    it 'raises an exception' do
+      expect {
+        TestWorker.perform_in(Time.now + 901, 'message')
+      }.to raise_error 'The maximum allowed delay is 15 minutes'
+    end
+  end
+
   describe '.perform_async' do
     it 'enqueues a message' do
-      expect(sqs_queue).to receive(:send_message).with('message', {})
+      expect(sqs_queue).to receive(:send_message).with('message', {
+        message_attributes: {
+          'shoryuken_class' => {
+            string_value: TestWorker.to_s,
+            data_type: 'String'
+          }
+        }
+      })
 
       TestWorker.perform_async('message')
     end
 
     it 'enqueues a message with options' do
-      expect(sqs_queue).to receive(:send_message).with('delayed message', delay_seconds: 60)
+      expect(sqs_queue).to receive(:send_message).with('delayed message', {
+        delay_seconds: 60,
+        message_attributes: {
+          'shoryuken_class' => {
+            string_value: TestWorker.to_s,
+            data_type: 'String'
+          }
+        }
+      })
 
       TestWorker.perform_async('delayed message', delay_seconds: 60)
     end
@@ -37,6 +96,7 @@ describe 'Shoryuken::Worker' do
       end
 
       expect(Shoryuken.workers['production_default']).to eq NewTestWorker
+      expect(NewTestWorker.get_shoryuken_options['queue']).to eq 'production_default'
     end
   end
 end

data/spec/shoryuken_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe Shoryuken do
+  describe '.register_worker' do
+    it 'registers a worker' do
+      described_class.workers.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.workers).to eq('default' => TestWorker)
+    end
+
+    it 'registers a batchable worker' do
+      described_class.workers.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.workers).to eq('default' => TestWorker)
+    end
+
+    it 'allows multiple workers' do
+      described_class.workers.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.workers).to eq('default' => TestWorker)
+
+      class Test2Worker
+        include Shoryuken::Worker
+
+        shoryuken_options queue: 'default'
+
+        def perform(sqs_msg, body); end
+      end
+
+      expect(described_class.workers).to eq('default' => Test2Worker)
+    end
+
+    it 'raises an exception when mixing batchable with non batchable' do
+      described_class.workers.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+
+      expect {
+        class BatchableWorker
+          include Shoryuken::Worker
+
+          shoryuken_options queue: 'default', batch: true
+
+          def perform(sqs_msg, body); end
+        end
+      }.to raise_error("Could not register BatchableWorker for 'default', because TestWorker is already registered for this queue, " \
+                       "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -4,6 +4,7 @@ Bundler.setup
 require 'pry-byebug'
 require 'celluloid'
 require 'shoryuken'
+require 'json'
 
 options_file = File.join(File.expand_path('../..', __FILE__), 'shoryuken.yml')
 
@@ -40,7 +41,7 @@ RSpec.configure do |config|
     Shoryuken.options.merge!($options)
 
     Shoryuken.queues.clear
-    Shoryuken.queues << 'shoryuken'
+
     Shoryuken.options[:concurrency] = 1
     Shoryuken.options[:delay]       = 1
     Shoryuken.options[:timeout]     = 1
@@ -49,5 +50,8 @@ RSpec.configure do |config|
 
     TestWorker.get_shoryuken_options.clear
     TestWorker.get_shoryuken_options['queue'] = 'default'
+
+    Shoryuken.workers.clear
+    Shoryuken.register_worker('default', TestWorker)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-26 00:00:00.000000000 Z
+date: 2014-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -84,16 +84,16 @@ dependencies:
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.15.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.15.2
 description: Shoryuken is a super efficient AWS SQS thread based message processor
 email:
 - pablo@pablocantero.com
@@ -110,10 +110,8 @@ files:
 - README.md
 - Rakefile
 - bin/shoryuken
-- examples/all.rb
+- examples/bootstrap_queues.rb
 - examples/default_worker.rb
-- examples/high_priority_worker.rb
-- examples/low_priority_worker.rb
 - lib/shoryuken.rb
 - lib/shoryuken/cli.rb
 - lib/shoryuken/client.rb
@@ -123,12 +121,14 @@ files:
 - lib/shoryuken/logging.rb
 - lib/shoryuken/manager.rb
 - lib/shoryuken/middleware/chain.rb
+- lib/shoryuken/middleware/server/active_record.rb
 - lib/shoryuken/middleware/server/auto_delete.rb
 - lib/shoryuken/middleware/server/timing.rb
 - lib/shoryuken/processor.rb
 - lib/shoryuken/util.rb
 - lib/shoryuken/version.rb
 - lib/shoryuken/worker.rb
+- lib/shoryuken/worker_loader.rb
 - shoryuken.gemspec
 - shoryuken.jpg
 - spec/integration/launcher_spec.rb
@@ -141,7 +141,9 @@ files:
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/util_spec.rb
+- spec/shoryuken/worker_loader_spec.rb
 - spec/shoryuken/worker_spec.rb
+- spec/shoryuken_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/phstc/shoryuken
 licenses:
@@ -178,5 +180,7 @@ test_files:
 - spec/shoryuken/middleware/server/timing_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/util_spec.rb
+- spec/shoryuken/worker_loader_spec.rb
 - spec/shoryuken/worker_spec.rb
+- spec/shoryuken_spec.rb
 - spec/spec_helper.rb

data/examples/all.rb

@@ -1,5 +0,0 @@
-$stdout.sync = true
-
-require_relative 'high_priority_worker'
-require_relative 'default_worker'
-require_relative 'low_priority_worker'

data/examples/high_priority_worker.rb

@@ -1,9 +0,0 @@
-class HighPriorityWorker
-  include Shoryuken::Worker
-
-  shoryuken_options queue: 'high_priority', delete: true
-
-  def perform(sqs_msg, body)
-    puts "HighPriorityWorker: '#{body}'"
-  end
-end

data/examples/low_priority_worker.rb

@@ -1,11 +0,0 @@
-class LowPriorityWorker
-  include Shoryuken::Worker
-
-  shoryuken_options queue: 'low_priority', delete: true, batch: true
-
-  def perform(sqs_msgs, bodies)
-    bodies.each_with_index do |body, index|
-      puts "LowPriorityWorker (#{index}): '#{body}'"
-    end
-  end
-end