shoryuken 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 720717277a58652fd893ef470426e5696b4943de
-  data.tar.gz: a5aad87ea2ffe866454cfa65ce78820c46def1a3
+  metadata.gz: 7391a170e4f76601238417e61b88861951bc6c73
+  data.tar.gz: 8983d19ef10bea4723533a12b23b74ca55562baf
 SHA512:
-  metadata.gz: 302578e60de657fafee3faf3952adf70c848cd5aec1d124f6e34db78e82269ad25bd629f6ce25737a674804a2223f85eff812ad6262c0c6d9ffb6524a08c20f2
-  data.tar.gz: 17966390a08f67d9830e804bc0f24b11efb0658648e515dce18d0c52ccd72ec60d0457c431ecf4fd4d0905d34e29fae3ca5fc3244548b25f562eda14cc5039b8
+  metadata.gz: 87a30b0002f3eaf8a62fd9f76e938fb57a5fcecc829c6a3cf8fdb7c4d1ab3388001fc460b92d59d3ae2762b441f447141bc92c136dbf6fb8730820c101df2dc0
+  data.tar.gz: 79f4d02f0dac764f23d07f8d27711c70b0cb97543cd410c2436300ff3e633a625de290caedf3746366b77d6c9b737c0c0138d000e2d610dab715267a40e74bb3

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+  # - 1.9.3
+  # - 1.9.2
+  - 2.0.0
+  - 2.1.0
+  # - ruby-head
+  # - jruby-19mode
+  # - jruby-head
+
+notifications:
+  email:
+    on_success: change
+    on_failure: always
+
+script: bundle exec rspec spec

data/README.md

@@ -4,6 +4,8 @@
 
 Shoryuken _sho-ryu-ken_ is a super efficient [AWS SQS](https://aws.amazon.com/sqs/) thread based message processor.
 
+[![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
+
 ## Key features
 
 ### Load balancing
@@ -34,10 +36,6 @@ If all queues get empty, all processors will be changed to the waiting state and
 
 To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches.
 
-## Resque compatible?
-
-Shoryuken isn't Resque compatible, it passes the [original SQS message](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/ReceivedMessage.html) to the workers.
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -60,8 +58,8 @@ Or install it yourself as:
 class HelloWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'hello', auto_delete: true
-  # shoryuken_options queue: ->{ "#{ENV['environment']_hello" }, auto_delete: true
+  shoryuken_options queue: 'hello', delete: true
+  # shoryuken_options queue: ->{ "#{ENV['environment']_hello" }, delete: true
 
   def perform(sqs_msg)
     puts "Hello #{sqs_msg.body}"
@@ -81,16 +79,16 @@ Sample configuration file `shoryuken.yml`.
 
 ```yaml
 aws:
-  access_key_id:      ...
-  secret_access_key:  ...
-  region:             us-east-1
-  receive_message:
+  access_key_id:      ...       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  secret_access_key:  ...       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+  region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
+  receive_message:              # See http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/Queue.html#receive_message-instance_method
+    # wait_time_seconds: N      # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
     attributes:
       - receive_count
       - sent_at
-concurrency: 25,
-delay: 25,
-timeout: 8
+concurrency: 25,  # The number of allocated threads to process messages. Default 25
+delay: 25,        # The delay in seconds to pause a queue when it's empty. Default 0
 queues:
   - [shoryuken, 6]
   - [uppercut, 2]
@@ -103,6 +101,25 @@ queues:
 bundle exec shoryuken -r worker.rb -C shoryuken.yml
 ```
 
+Other options:
+
+```bash
+shoryuken --help
+
+shoryuken [options]
+    -c, --concurrency INT            Processor threads to use
+    -d, --daemon                     Daemonize process
+    -q, --queue QUEUE[,WEIGHT]...    Queues to process with optional weights
+    -r, --require [PATH|DIR]         Location of the worker
+    -C, --config PATH                Path to YAML config file
+    -L, --logfile PATH               Path to writable logfile
+    -P, --pidfile PATH               Path to pidfile
+    -v, --verbose                    Print more verbose output
+    -V, --version                    Print version and exit
+    -h, --help                       Show help
+    ...
+```
+
 ### Middleware
 
 ```ruby
@@ -122,6 +139,10 @@ Shoryuken.configure_server do |config|
 end
 ```
 
+## More Information
+
+Please check the [Shoryuken Wiki](https://github.com/phstc/shoryuken/wiki).
+
 ## Credits
 
 [Mike Perham](https://github.com/mperham), creator of [Sidekiq](https://github.com/mperham/sidekiq), and [everybody who contributed to it](https://github.com/mperham/sidekiq/graphs/contributors). Shoryuken wouldn't exist as it is without those contributions.

data/Rakefile

@@ -1,6 +1,7 @@
 require 'bundler/gem_tasks'
 $stdout.sync = true
 
+desc 'Open Shoryuken pry console'
 task :console do
   require 'pry'
   require 'shoryuken'
@@ -21,6 +22,7 @@ task :console do
   Pry.start
 end
 
+desc 'Push test messages to shoryuken, uppercut and sidekiq'
 task :push_test, :size do |t, args|
   require 'yaml'
   require 'shoryuken'

data/bin/shoryuken

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'shoryuken'
-require 'shoryuken/cli'
+require_relative '../lib/shoryuken/cli'
 
 begin
   Shoryuken::CLI.instance.run(ARGV)

data/examples/shoryuken_worker.rb

@@ -1,7 +1,7 @@
 class ShoryukenWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'shoryuken', auto_delete: true
+  shoryuken_options queue: 'shoryuken', delete: true
 
   def perform(sqs_msg)
     puts "Shoryuken: '#{sqs_msg.body}'"

data/examples/sidekiq_worker.rb

@@ -1,7 +1,7 @@
 class SidekiqWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'sidekiq', auto_delete: true
+  shoryuken_options queue: 'sidekiq', delete: true
 
   def perform(sqs_msg)
     puts "Sidekiq: '#{sqs_msg.body}'"

data/examples/uppercut_worker.rb

@@ -1,7 +1,7 @@
 class UppercutWorker
   include Shoryuken::Worker
 
-  shoryuken_options queue: 'uppercut', auto_delete: true
+  shoryuken_options queue: 'uppercut', delete: true
 
   def perform(sqs_msg)
     puts "Uppercut: '#{sqs_msg.body}'"

data/lib/shoryuken/cli.rb

@@ -4,6 +4,8 @@ require 'singleton'
 require 'optparse'
 require 'erb'
 
+require 'shoryuken'
+
 module Shoryuken
   class CLI
     include Util
@@ -24,10 +26,11 @@ module Shoryuken
       initialize_logger
       validate!
       daemonize
-      initialize_aws
-      require_workers
       write_pid
+      load_celluloid
+      require_workers
 
+      require 'shoryuken/launcher'
       @launcher = Shoryuken::Launcher.new
 
       begin
@@ -45,6 +48,18 @@ module Shoryuken
 
     private
 
+    def load_celluloid
+      raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization" if defined?(::Celluloid) && Shoryuken.options[:daemon]
+
+      # Celluloid can't be loaded until after we've daemonized
+      # because it spins up threads and creates locks which get
+      # into a very bad state if forked.
+      require 'celluloid/autostart'
+      Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
+
+      require 'shoryuken/manager'
+    end
+
     def daemonize
       return unless Shoryuken.options[:daemon]
 
@@ -88,7 +103,7 @@ module Shoryuken
       opts = {}
 
       @parser = OptionParser.new do |o|
-        o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
+        o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
           opts[:concurrency] = Integer(arg)
         end
 
@@ -97,23 +112,23 @@ module Shoryuken
         end
 
         o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
-          queues_and_weights = arg.scan(/([\w\.-]+),?(\d*)/)
-          parse_queues opts, queues_and_weights
+          queue, weight = arg.split(',')
+          parse_queue queue, weight
         end
 
         o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
           opts[:require] = arg
         end
 
-        o.on '-C', '--config PATH', 'path to YAML config file' do |arg|
+        o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
           opts[:config_file] = arg
         end
 
-        o.on '-L', '--logfile PATH', 'path to writable logfile' do |arg|
+        o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
           opts[:logfile] = arg
         end
 
-        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+        o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
           opts[:pidfile] = arg
         end
 
@@ -156,15 +171,11 @@ module Shoryuken
           end
         end
 
-        Shoryuken.logger.info "Ready processors: #{launcher.manager.instance_variable_get(:@ready).size}"
-        Shoryuken.logger.info "Busy processors: #{launcher.manager.instance_variable_get(:@busy).size}"
+        ready  = launcher.manager.instance_variable_get(:@ready).size
+        busy   = launcher.manager.instance_variable_get(:@busy).size
+        queues = launcher.manager.instance_variable_get(:@queues)
 
-        launcher.manager.instance_variable_get(:@queues).inject({}) do |weights, queue|
-          weights[queue] = weights[queue].to_i + 1
-          weights
-        end.each do |queue, weight|
-          Shoryuken.logger.info "Current queue '#{queue}' weight: #{weight}"
-        end
+        Shoryuken.logger.info "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}"
       else
         Shoryuken.logger.info "Received #{sig}, will shutdown down"
 
@@ -184,13 +195,12 @@ module Shoryuken
       parse_queues
     end
 
-    def parse_config(cfile)
-      opts = {}
-      if File.exist?(cfile)
-        opts = YAML.load(ERB.new(IO.read(cfile)).result)
+    def parse_config(config_file)
+      if File.exist?(config_file)
+        YAML.load(ERB.new(IO.read(config_file)).result)
+      else
+        raise ArgumentError, "Config file #{config_file} does not exist"
       end
-
-      opts
     end
 
     def initialize_logger
@@ -200,10 +210,32 @@ module Shoryuken
     end
 
     def validate!
+      if Shoryuken.options[:aws][:access_key_id].nil? && Shoryuken.options[:aws][:secret_access_key].nil?
+        if ENV['AWS_ACCESS_KEY_ID'].nil? && ENV['AWS_SECRET_ACCESS_KEY'].nil?
+          raise ArgumentError, 'No AWS credentials supplied'
+        end
+      end
+
+
+      initialize_aws
+
+      Shoryuken.queues.uniq.each do |queue|
+        # validate all queues and AWS credentials consequently
+        begin
+          Shoryuken::Client.queues queue
+        rescue AWS::SQS::Errors::NonExistentQueue => e
+          raise ArgumentError, "Queue '#{queue}' does not exist"
+        rescue => e
+          raise
+        end
+      end
+
       raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
     end
 
     def initialize_aws
+      # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+      # when not explicit supplied
       AWS.config Shoryuken.options[:aws] if Shoryuken.options[:aws]
     end
 
@@ -212,7 +244,7 @@ module Shoryuken
     end
 
     def parse_queues
-      Shoryuken.options[:queues].each { |queue_and_weight| parse_queue *queue_and_weight }
+      Shoryuken.options[:queues].to_a.each { |queue_and_weight| parse_queue *queue_and_weight }
     end
 
     def parse_queue(queue, weight = nil)

data/lib/shoryuken/client.rb

@@ -1,23 +1,24 @@
-
 module Shoryuken
   class Client
     @@queues = {}
+    @@visibility_timeouts = {}
 
-    def self.queues(name)
-      @@queues[name.to_s] ||= sqs.queues.named(name)
-    end
+    class << self
+      def queues(queue)
+        @@queues[queue.to_s] ||= sqs.queues.named(queue)
+      end
 
-    def self.receive_message(queue, options = {})
-      queues(queue).receive_message(Hash(options))
-    end
+      def visibility_timeout(queue)
+        @@visibility_timeouts[queue.to_s] ||= queues(queue).visibility_timeout
+      end
 
-    def self.reset!
-      # for test purposes
-      @@queues = {}
-    end
+      def receive_message(queue, options = {})
+        queues(queue).receive_message(Hash(options))
+      end
 
-    def self.sqs
-      @sqs ||= AWS::SQS.new
+      def sqs
+        @sqs ||= AWS::SQS.new
+      end
     end
   end
 end

data/lib/shoryuken/fetcher.rb

@@ -3,13 +3,15 @@ module Shoryuken
     include Celluloid
     include Util
 
+    FETCH_LIMIT = 10
+
     def initialize(manager)
       @manager = manager
     end
 
     def receive_message(queue, limit)
       # AWS limits the batch size by 10
-      limit = limit > 10 ? 10 : limit
+      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
       Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
     end
@@ -18,32 +20,51 @@ module Shoryuken
       watchdog('Fetcher#fetch died') do
         started_at = Time.now
 
-        logger.info "Looking for new messages queue '#{queue}'"
+        logger.info "Looking for new messages '#{queue}'"
 
         begin
-          if (sqs_msgs = Array(receive_message(queue, available_processors))).any?
-            logger.info "Message found for queue '#{queue}'"
+          batch = !!Shoryuken.workers[queue].get_shoryuken_options['batch']
+
+          limit = batch ? FETCH_LIMIT : available_processors
+
+          if (sqs_msgs = Array(receive_message(queue, limit))).any?
+            logger.info "Found #{sqs_msgs.size} messages for '#{queue}'"
 
-            sqs_msgs.each do |sqs_msg|
-              @manager.async.rebalance_queue_weight!(queue)
-              @manager.async.assign(queue, sqs_msg)
+            if batch
+              @manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
+            else
+              sqs_msgs.each { |sqs_msg| @manager.async.assign(queue, sqs_msg) }
             end
+
+            @manager.async.rebalance_queue_weight!(queue)
           else
-            logger.info "No message found for queue '#{queue}'"
+            logger.info "No message found for '#{queue}'"
 
             @manager.async.pause_queue!(queue)
           end
 
           @manager.async.dispatch
 
-          logger.debug "Fetcher#fetch('#{queue}') completed in #{elapsed(started_at)} ms"
+          logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
         rescue => ex
-          logger.error("Error fetching message: #{ex}")
-          logger.error(ex.backtrace.first)
+          logger.error "Error fetching message: #{ex}"
+          logger.error ex.backtrace.first
 
           @manager.async.dispatch
         end
       end
+
+    end
+    private
+
+    def patch_sqs_msgs!(sqs_msgs)
+      sqs_msgs.instance_eval do
+        def id
+          "batch-with-#{size}-messages"
+        end
+      end
+
+      sqs_msgs
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -1,9 +1,3 @@
-require 'yaml'
-require 'aws-sdk'
-require 'celluloid'
-
-require 'shoryuken/version'
-require 'shoryuken/manager'
 require 'shoryuken/processor'
 require 'shoryuken/fetcher'
 
@@ -57,7 +51,7 @@ module Shoryuken
 
     def processor_done(queue, processor)
       watchdog('Manager#processor_done died') do
-        logger.info "Process done for queue '#{queue}'"
+        logger.info "Process done for '#{queue}'"
 
         @busy.delete processor
 
@@ -99,11 +93,7 @@ module Shoryuken
     def rebalance_queue_weight!(queue)
       watchdog('Manager#rebalance_queue_weight! died') do
         if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
-          if current + 1 > original
-            logger.info "Increasing queue '#{queue}' weight to #{current + 1}, original: #{original}"
-          else
-            logger.info "Queue '#{queue}' is back to its normal weight #{original}"
-          end
+          logger.info "Increasing '#{queue}' weight to #{current + 1}, max: #{original}"
 
           @queues << queue
         end
@@ -111,9 +101,9 @@ module Shoryuken
     end
 
     def pause_queue!(queue)
-      return unless @queues.include? queue
+      return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
 
-      logger.info "Pausing queue '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because the queue is empty"
+      logger.info "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
 
       @queues.delete(queue)
 
@@ -124,9 +114,7 @@ module Shoryuken
     def dispatch
       return if stopped?
 
-      logger.debug { "Ready size: #{@ready.size}" }
-      logger.debug { "Busy size: #{@busy.size}" }
-      logger.debug { "Queues: #{@queues.inspect}" }
+      logger.debug { "Ready: #{@ready.size}, Busy: #{@busy.size}, Active Queues: #{unparse_queues(@queues)}" }
 
       if @ready.empty?
         logger.debug { 'Pausing fetcher, because all processors are busy' }
@@ -151,7 +139,7 @@ module Shoryuken
       return if stopped?
 
       unless @queues.include? queue
-        logger.info "Restarting queue '#{queue}'"
+        logger.info "Restarting '#{queue}'"
 
         @queues << queue
 

data/lib/shoryuken/middleware/server/delete.rb

@@ -0,0 +1,17 @@
+module Shoryuken
+  module Middleware
+    module Server
+      class Delete
+        def call(worker, queue, sqs_msg)
+          yield
+
+          # auto_delete is deprecated
+          delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
+
+          Array(sqs_msg).each(&:delete) if delete
+        end
+      end
+    end
+  end
+end
+

data/lib/shoryuken/middleware/server/logging.rb

@@ -5,15 +5,24 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg)
-          Shoryuken::Logging.with_context("#{worker.class.to_s} '#{queue}' '#{sqs_msg.id}'") do
+          Shoryuken::Logging.with_context("#{worker.class.to_s}/#{queue}/#{sqs_msg.id}") do
             begin
               started_at = Time.now
+
               logger.info { "started at #{started_at}" }
+
               yield
-              logger.info { "completed in: #{elapsed(started_at)} ms" }
-            rescue Exception
+
+              total_time = elapsed(started_at)
+
+              if (total_time / 1000.0) > (timeout = Shoryuken::Client.visibility_timeout(queue))
+                logger.warn "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms"
+              end
+
+              logger.info { "completed in: #{total_time} ms" }
+            rescue => e
               logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise
+              raise e
             end
           end
         end

data/lib/shoryuken/processor.rb

@@ -24,13 +24,5 @@ module Shoryuken
 
       @manager.async.processor_done(queue, current_actor)
     end
-
-    def self.default_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Logging
-        m.add Middleware::Server::AutoDelete
-        # m.add Middleware::Server::RetryJobs
-      end
-    end
   end
 end

data/lib/shoryuken/util.rb

@@ -15,5 +15,12 @@ module Shoryuken
     def elapsed(started_at)
       (Time.now - started_at) * 1000
     end
+
+    def unparse_queues(queues)
+      queues.inject({}) do |queue_and_weights, name|
+        queue_and_weights[name] = queue_and_weights[name].to_i + 1
+        queue_and_weights
+      end.to_a
+    end
   end
 end

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/shoryuken/worker.rb

@@ -10,12 +10,11 @@ module Shoryuken
         queue = @shoryuken_options['queue']
         queue = queue.call if queue.respond_to? :call
 
-
         Shoryuken.register_worker(queue, self)
       end
 
       def get_shoryuken_options # :nodoc:
-        @shoryuken_options || { 'queue' => 'default' }
+        @shoryuken_options || { 'queue' => 'default', 'delete' => false, 'batch' => false }
       end
 
       def stringify_keys(hash) # :nodoc:

data/lib/shoryuken.rb

@@ -1,35 +1,27 @@
 require 'yaml'
 require 'aws-sdk'
-require 'celluloid'
 require 'time'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
 require 'shoryuken/util'
-require 'shoryuken/manager'
-require 'shoryuken/processor'
-require 'shoryuken/fetcher'
 require 'shoryuken/client'
 require 'shoryuken/worker'
-require 'shoryuken/launcher'
 require 'shoryuken/logging'
 require 'shoryuken/middleware/chain'
-require 'shoryuken/middleware/server/auto_delete'
+require 'shoryuken/middleware/server/delete'
 require 'shoryuken/middleware/server/logging'
 
 module Shoryuken
   DEFAULTS = {
     concurrency: 25,
     queues: [],
-    receive_message_options: {},
-    delay: 25,
+    aws: {},
+    delay: 0,
     timeout: 8
   }
 
-  # { 'my_queue1' => Worker1
-  #   'my_queue2' => Worker2 }
   @@workers = {}
-
   @@queues = []
 
   def self.options
@@ -62,8 +54,19 @@ module Shoryuken
   end
 
   def self.server_middleware
-    @server_chain ||= Processor.default_middleware
+    @server_chain ||= default_server_middleware
     yield @server_chain if block_given?
     @server_chain
   end
+
+
+  private
+
+  def self.default_server_middleware
+    Middleware::Chain.new do |m|
+      m.add Middleware::Server::Logging
+      m.add Middleware::Server::Delete
+      # TODO m.add Middleware::Server::RetryJobs
+    end
+  end
 end

data/spec/integration/launcher_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+require 'shoryuken/manager'
+require 'shoryuken/launcher'
+
+describe Shoryuken::Launcher do
+  describe 'Consuming messages', slow: :true do
+    before do
+      Shoryuken.options[:aws] ||= {}
+      Shoryuken.options[:aws][:receive_message] ||= {}
+      Shoryuken.options[:aws][:receive_message][:wait_time_seconds] = 5
+
+      subject.run
+
+      $received_messages = 0
+    end
+
+    after do
+      subject.stop
+    end
+
+    class ShoryukenWorker
+      include Shoryuken::Worker
+
+      shoryuken_options queue: 'shoryuken', delete: true
+
+      def perform(sqs_msg)
+        $received_messages = Array(sqs_msg).size
+      end
+    end
+
+    it 'consumes a message' do
+      ShoryukenWorker.get_shoryuken_options['batch'] = false
+
+      Shoryuken::Client.queues('shoryuken').send_message('Yo')
+
+      10.times do
+        break if $received_messages > 0
+        sleep 0.2
+      end
+
+      expect($received_messages).to eq 1
+    end
+
+    it 'consumes a batch' do
+      ShoryukenWorker.get_shoryuken_options['batch'] = true
+
+      Shoryuken::Client.queues('shoryuken').batch_send *(['Yo'] * 10)
+
+      10.times do
+        break if $received_messages > 0
+        sleep 0.2
+      end
+
+      # the fetch result is uncertain, should be greater than 1, but hard to tell the exact size
+      expect($received_messages).to be > 1
+    end
+  end
+end

data/spec/shoryuken/client_spec.rb

@@ -3,20 +3,30 @@ require 'spec_helper'
 describe Shoryuken::Client do
   let(:sqs)              { double 'SQS' }
   let(:queue_collection) { double 'Queues Collection' }
-  let(:queue)            { double 'Queue' }
-  let(:queue_name)       { 'shoryuken' }
+  let(:sqs_queue)        { double 'SQS Queue' }
+  let(:queue)            { 'shoryuken' }
 
   before do
     allow(described_class).to receive(:sqs).and_return(sqs)
     allow(sqs).to receive(:queues).and_return(queue_collection)
+    allow(queue_collection).to receive(:named).and_return(sqs_queue)
   end
 
   describe '.queues' do
     it 'memoizes queues' do
-      expect(queue_collection).to receive(:named).once.with(queue_name).and_return(queue)
+      expect(queue_collection).to receive(:named).once.with(queue).and_return(sqs_queue)
 
-      expect(Shoryuken::Client.queues(queue_name)).to eq queue
-      expect(Shoryuken::Client.queues(queue_name)).to eq queue
+      expect(Shoryuken::Client.queues(queue)).to eq sqs_queue
+      expect(Shoryuken::Client.queues(queue)).to eq sqs_queue
+    end
+  end
+
+  describe '.visibility_timeout' do
+    it 'memoizes visibility_timeout' do
+      expect(sqs_queue).to receive(:visibility_timeout).once.and_return(30)
+
+      expect(Shoryuken::Client.visibility_timeout(queue)).to eq 30
+      expect(Shoryuken::Client.visibility_timeout(queue)).to eq 30
     end
   end
 end

data/spec/shoryuken/fetcher_spec.rb

@@ -1,9 +1,11 @@
 require 'spec_helper'
+require 'shoryuken/manager'
+require 'shoryuken/fetcher'
 
 describe Shoryuken::Fetcher do
   let(:manager)   { double Shoryuken::Manager }
   let(:sqs_queue) { double 'sqs_queue' }
-  let(:queue)     { 'shoryuken' }
+  let(:queue)     { 'shoryuken2' }
   let(:sqs_msg)   { double 'SQS msg'}
 
   subject { described_class.new(manager) }
@@ -13,8 +15,17 @@ describe Shoryuken::Fetcher do
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
+
+  class Shoryuken2Worker
+    include Shoryuken::Worker
+
+    shoryuken_options queue: 'shoryuken2'
+
+    def perform(sqs_msg); end
+  end
+
   describe '#fetch' do
-    it 'calls pause_queue! when not found' do
+    it 'calls pause when no message' do
       allow(sqs_queue).to receive(:receive_message).with(limit: 1).and_return([])
 
       expect(manager).to receive(:pause_queue!).with(queue)
@@ -32,5 +43,17 @@ describe Shoryuken::Fetcher do
 
       subject.fetch(queue, 5)
     end
+
+    it 'assigns messages in batch' do
+      Shoryuken2Worker.get_shoryuken_options['batch'] = true
+
+      allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT).and_return(sqs_msg)
+
+      expect(manager).to receive(:rebalance_queue_weight!).with(queue)
+      expect(manager).to receive(:assign).with(queue, [sqs_msg])
+      expect(manager).to receive(:dispatch)
+
+      subject.fetch(queue, 5)
+    end
   end
 end

data/spec/shoryuken/manager_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'shoryuken/manager'
 
 describe Shoryuken::Manager do
   describe 'Auto Scaling' do

data/spec/shoryuken/middleware/server/delete_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe Shoryuken::Middleware::Server::Delete do
+  let(:sqs_msg) { double 'SQS msg' }
+
+  before do
+    DeleteWorker.get_shoryuken_options['delete'] = true
+  end
+
+  class DeleteWorker
+    include Shoryuken::Worker
+
+    shoryuken_options queue: 'delete', delete: true
+
+    def perform; end
+  end
+
+  it 'deletes a message' do
+    expect(sqs_msg).to receive(:delete)
+
+    subject.call(DeleteWorker.new, 'delete', sqs_msg) {}
+  end
+
+  it 'deletes a batch' do
+    sqs_msg2 = double
+    sqs_msg3 = double
+
+    expect(sqs_msg).to receive(:delete)
+    expect(sqs_msg2).to receive(:delete)
+    expect(sqs_msg3).to receive(:delete)
+
+    subject.call(DeleteWorker.new, 'delete', [sqs_msg, sqs_msg2, sqs_msg3]) {}
+  end
+
+  it 'does not delete a message' do
+    DeleteWorker.get_shoryuken_options['delete'] = false
+
+    expect(sqs_msg).to_not receive(:delete)
+
+    subject.call(DeleteWorker.new, 'delete', sqs_msg) {}
+  end
+
+  context 'when exception' do
+    it 'does not delete a message' do
+      expect(sqs_msg).to_not receive(:delete)
+
+      expect {
+        subject.call(DeleteWorker.new, 'delete', sqs_msg) { raise }
+      }.to raise_error
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -1,8 +1,9 @@
 require 'spec_helper'
+require 'shoryuken/processor'
 
 describe Shoryuken::Processor do
   let(:manager)   { double Shoryuken::Manager }
-  let(:sqs_queue) { double 'Queue' }
+  let(:sqs_queue) { double 'Queue', visibility_timeout: 30 }
   let(:queue)     { 'yo' }
   let(:sqs_msg)   { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e' }
 
@@ -66,8 +67,8 @@ describe Shoryuken::Processor do
       end
     end
 
-    it 'performs with auto delete' do
-      YoWorker.get_shoryuken_options['auto_delete'] = true
+    it 'performs with delete' do
+      YoWorker.get_shoryuken_options['delete'] = true
 
       expect(manager).to receive(:processor_done).with(queue, subject)
 
@@ -78,8 +79,8 @@ describe Shoryuken::Processor do
       subject.process(queue, sqs_msg)
     end
 
-    it 'performs without auto delete' do
-      YoWorker.get_shoryuken_options['auto_delete'] = false
+    it 'performs without delete' do
+      YoWorker.get_shoryuken_options['delete'] = false
 
       expect(manager).to receive(:processor_done).with(queue, subject)
 

data/spec/shoryuken/util_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe 'Shoryuken::Util' do
+  subject do
+    Class.new do
+      extend Shoryuken::Util
+    end
+  end
+
+  describe '#unparse_queues' do
+    it 'returns queues and weights' do
+      queues = %w[queue1 queue1 queue2 queue3 queue4 queue4 queue4]
+
+      expect(subject.unparse_queues(queues)).to eq([['queue1', 2], ['queue2', 1], ['queue3', 1], ['queue4', 3]])
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -3,6 +3,7 @@ Bundler.setup
 
 require 'pry-byebug'
 require 'shoryuken'
+require 'celluloid'
 
 options_file = File.join(File.expand_path('../..', __FILE__), 'shoryuken.yml')
 
@@ -24,7 +25,8 @@ RSpec.configure do |config|
     # remove doubles, preventing:
     # Double "Queue" was originally created in one example but has leaked into another example and can no longer be used.
     # rspec-mocks' doubles are designed to only last for one example, and you need to create a new one in each example you wish to use it for.
-    Shoryuken::Client.reset!
+    Shoryuken::Client.class_variable_set :@@queues, {}
+    Shoryuken::Client.class_variable_set :@@visibility_timeouts, {}
 
     Shoryuken.options.clear
     Shoryuken.options.merge!($options)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-06 00:00:00.000000000 Z
+date: 2014-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -118,6 +118,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -136,7 +137,7 @@ files:
 - lib/shoryuken/logging.rb
 - lib/shoryuken/manager.rb
 - lib/shoryuken/middleware/chain.rb
-- lib/shoryuken/middleware/server/auto_delete.rb
+- lib/shoryuken/middleware/server/delete.rb
 - lib/shoryuken/middleware/server/logging.rb
 - lib/shoryuken/processor.rb
 - lib/shoryuken/util.rb
@@ -144,13 +145,15 @@ files:
 - lib/shoryuken/worker.rb
 - shoryuken.gemspec
 - shoryuken.jpg
-- spec/shoryuken/chain_spec.rb
+- spec/integration/launcher_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/fetcher_spec.rb
-- spec/shoryuken/integration/launcher_spec.rb
 - spec/shoryuken/manager_spec.rb
+- spec/shoryuken/middleware/chain_spec.rb
+- spec/shoryuken/middleware/server/delete_spec.rb
 - spec/shoryuken/processor_spec.rb
+- spec/shoryuken/util_spec.rb
 - spec/shoryuken/worker_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/phstc/shoryuken
@@ -178,12 +181,14 @@ signing_key:
 specification_version: 4
 summary: Shoryuken is a super efficient AWS SQS thread based message processor
 test_files:
-- spec/shoryuken/chain_spec.rb
+- spec/integration/launcher_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/fetcher_spec.rb
-- spec/shoryuken/integration/launcher_spec.rb
 - spec/shoryuken/manager_spec.rb
+- spec/shoryuken/middleware/chain_spec.rb
+- spec/shoryuken/middleware/server/delete_spec.rb
 - spec/shoryuken/processor_spec.rb
+- spec/shoryuken/util_spec.rb
 - spec/shoryuken/worker_spec.rb
 - spec/spec_helper.rb

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -1,14 +0,0 @@
-module Shoryuken
-  module Middleware
-    module Server
-      class AutoDelete
-        def call(worker, queue, sqs_msg)
-          yield
-
-          sqs_msg.delete if worker.class.get_shoryuken_options['auto_delete']
-        end
-      end
-    end
-  end
-end
-

data/spec/shoryuken/integration/launcher_spec.rb

@@ -1,36 +0,0 @@
-require 'spec_helper'
-
-describe Shoryuken::Launcher do
-  describe 'Consuming messages', slow: :true do
-    before do
-      subject.run
-
-      $received_messages = 0
-    end
-
-    after do
-      subject.stop
-    end
-
-    class ShoryukenWorker
-      include Shoryuken::Worker
-
-      shoryuken_options queue: 'shoryuken', auto_delete: true
-
-      def perform(sqs_msg)
-        $received_messages += 1
-      end
-    end
-
-    it 'consumes a message' do
-      Shoryuken::Client.queues('shoryuken').send_message('Yo')
-
-      10.times do
-        break if $received_messages > 0
-        sleep 0.5
-      end
-
-      expect($received_messages).to eq 1
-    end
-  end
-end