shoryuken 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70a39ae890a6fe9dcff44b0c6ceeab9c77931ab2
-  data.tar.gz: ff223f629ed2f60cc0abb32fa17f28e15d680d2f
+  metadata.gz: 93f2193b2c553192ca3d38d1dbcd981bbaa35eeb
+  data.tar.gz: d46dcc643bb7fffeb78f4e2f6ca7f4e1b236d688
 SHA512:
-  metadata.gz: f0afdb29904c40200d1976f2b26973ae6694e7018b07bf133153c355e250a1d613cf1fd9c0500c2aaa612d77f4c5d631823ab186642b4a2051a76ea78feea4e6
-  data.tar.gz: 8b3d0dc3f255b3d921386a322e645e7875f2962c902094a48671d24a95211b02e5f0558be53e30619baf92551e9f3d7e23b60def11a2c11b84107525e339f141
+  metadata.gz: 5827e0327eba8f21ab4156d197ea0fd5a8587aa86afda60d14ffc9269e5606aa9688060d59d711821bc117d69ae49a41c0837aaea09433dbb3081fc66b36b9d3
+  data.tar.gz: fcc6b86772574af993cf9145f4d1b0f5bd3e4900937f13c2328ab7dd292dc9bbe1c9e502ea5826220734f091a6ae9d759c0ec1e836c10ce85858925071d94f01

data/README.md

@@ -2,7 +2,7 @@
 
 ![](shoryuken.jpg)
 
-Shoryuken _sho-ryu-ken_ is a super efficient [AWS SQS](https://aws.amazon.com/sqs/) thread based message processor.
+Shoryuken _sho-ryu-ken_ is a super-efficient [AWS SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
 
@@ -41,11 +41,15 @@ To be even more performance and cost efficient, Shoryuken fetches SQS messages i
 
 Add this line to your application's Gemfile:
 
-    gem 'shoryuken'
+```ruby
+gem 'shoryuken'
+```
 
 Or to get the latest updates:
 
-    gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
+```ruby
+gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
+```
 
 And then execute:
 
@@ -64,7 +68,7 @@ class MyWorker
   include Shoryuken::Worker
 
   shoryuken_options queue: 'default', auto_delete: true
-  # shoryuken_options queue: ->{ "#{ENV['environment']_default" }
+  # shoryuken_options queue: ->{ "#{ENV['environment']}_default" }
 
   # shoryuken_options body_parser: :json
   # shoryuken_options body_parser: ->(sqs_msg){ REXML::Document.new(sqs_msg.body) }
@@ -82,7 +86,7 @@ end
 
 [Check the Sending a message documentation](https://github.com/phstc/shoryuken/wiki/Sending-a-message)
 
-### Midleware
+### Middleware
 
 ```ruby
 class MyMiddleware
@@ -124,8 +128,52 @@ You can tell Shoryuken to load your Rails application by passing the `-R` or `--
 
 If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
 
+For middleware and other configuration, you might want to create an initializer:
+
+```ruby
+Shoryuken.configure_server do |config|
+  # Replace Rails logger so messages are logged wherever Shoryuken is logging
+  # Note: this entire block is only run by the processor, so we don't overwrite
+  #       the logger when the app is running as usual.
+  Rails.logger = Shoryuken::Logging.logger
+
+  config.server_middleware do |chain|
+    chain.add Shoryuken::MyMiddleware
+  end
+end
+```
+
+*Note:* In the above case, since we are replacing the Rails logger, it's desired that this initializer runs before other initializers (in case they themselves use the logger). Since by Rails conventions initializers are executed in alphabetical order, this can be achieved by prepending the initializer filename with `00_` (assuming no other initializers alphabetically precede this one).
+
 This feature works for Rails 4+, but needs to be confirmed for older versions.
 
+#### ActiveJob Support
+
+Yes, Shoryuken supports ActiveJob! This means that you can put your jobs in processor-agnostic `ActiveJob::Base` subclasses, and change processors whenever you want (or better yet, switch to Shoryuken from another processor easily!).
+
+It works as expected. Just put your job in `app/jobs`. Here's an example:
+
+```ruby
+class ProcessPhotoJob < ActiveJob::Base
+  queue_as :default
+
+  rescue_from ActiveJob::DeserializationError do |e|
+    Shoryuken.logger.error ex
+    Shoryuken.logger.error ex.backtrace.join("\n")
+  end
+
+  def perform(photo)
+    photo.process_image!
+  end
+end
+```
+
+Delayed mailers, ActiveRecord serialization, etc. all work.
+
+See [ActiveJob docs](http://edgeguides.rubyonrails.org/active_job_basics.html) for more info.
+
+*Note:* When queueing jobs to be performed in the future (e.g when setting the `wait` or `wait_until` ActiveJob options), SQS limits the amount of time to 15 minutes. Shoryuken will raise an exception if you attempt to schedule a job further into the future than this limit.
+
 ### Start Shoryuken
 
 ```shell

data/lib/shoryuken.rb

@@ -7,7 +7,8 @@ require 'shoryuken/core_ext'
 require 'shoryuken/util'
 require 'shoryuken/client'
 require 'shoryuken/worker'
-require 'shoryuken/worker_loader'
+require 'shoryuken/worker_registry'
+require 'shoryuken/default_worker_registry'
 require 'shoryuken/logging'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
@@ -22,31 +23,14 @@ module Shoryuken
     timeout: 8
   }
 
-  @@workers       = {}
-  @@queues        = []
-  @@worker_loader = WorkerLoader
+  @@queues          = []
+  @@worker_registry = DefaultWorkerRegistry.new
 
   class << self
     def options
       @options ||= DEFAULTS.dup
     end
 
-    def register_worker(queue, clazz)
-      if worker_class = @@workers[queue]
-        if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
-            "because #{worker_class} is already registered for this queue, "\
-            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
-        end
-      end
-
-      @@workers[queue] = clazz
-    end
-
-    def workers
-      @@workers
-    end
-
     def queues
       @@queues
     end
@@ -55,12 +39,16 @@ module Shoryuken
       Shoryuken::Logging.logger
     end
 
-    def worker_loader=(worker_loader)
-      @@worker_loader = worker_loader
+    def register_worker(*args)
+      worker_registry.register_worker(*args)
     end
 
-    def worker_loader
-      @@worker_loader
+    def worker_registry=(worker_registry)
+      @@worker_registry = worker_registry
+    end
+
+    def worker_registry
+      @@worker_registry
     end
 
     # Shoryuken.configure_server do |config|
@@ -78,6 +66,18 @@ module Shoryuken
       @server_chain
     end
 
+    def default_worker_options
+      @@default_worker_options ||= {
+        'queue'                   => 'default',
+        'delete'                  => false,
+        'auto_delete'             => false,
+        'auto_visibility_timeout' => false,
+        'batch'                   => false }
+    end
+
+    def default_worker_options=(options)
+      @@default_worker_options = options
+    end
 
     private
 
@@ -89,8 +89,9 @@ module Shoryuken
           require 'shoryuken/middleware/server/active_record'
           m.add Middleware::Server::ActiveRecord
         end
-        # TODO m.add Middleware::Server::RetryJobs
       end
     end
   end
 end
+
+require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)

data/lib/shoryuken/cli.rb

@@ -7,6 +7,9 @@ require 'erb'
 require 'shoryuken'
 
 module Shoryuken
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
   class CLI
     include Util
     include Singleton
@@ -24,9 +27,9 @@ module Shoryuken
 
       setup_options(args) do |cli_options|
         # this needs to happen before configuration is parsed, since it may depend on Rails env
+        initialize_logger(cli_options)
         load_rails if cli_options[:rails]
       end
-      initialize_logger
       require_workers
       validate!
       patch_deprecated_workers!
@@ -77,6 +80,7 @@ module Shoryuken
         ::Rails::Application.initializer "shoryuken.eager_load" do
           ::Rails.application.config.eager_load = true
         end
+        require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
         require File.expand_path("config/environment.rb")
       end
 
@@ -233,22 +237,21 @@ module Shoryuken
       end
     end
 
-    def initialize_logger
-      Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
+    def initialize_logger(options = Shoryuken.options)
+      Shoryuken::Logging.initialize_logger(options[:logfile]) if options[:logfile]
 
-      Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
+      Shoryuken.logger.level = Logger::DEBUG if options[:verbose]
     end
 
     def validate!
       raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
 
-      Shoryuken.queues.each do |queue|
-        logger.warn "No worker supplied for '#{queue}'" unless Shoryuken.workers.include? queue
-      end
+      all_queues = Shoryuken.queues
+      queues_with_workers = Shoryuken.worker_registry.queues
 
-      if Shoryuken.options[:aws][:access_key_id].nil? && Shoryuken.options[:aws][:secret_access_key].nil?
-        if ENV['AWS_ACCESS_KEY_ID'].nil? && ENV['AWS_SECRET_ACCESS_KEY'].nil?
-          raise ArgumentError, 'No AWS credentials supplied'
+      unless defined?(::ActiveJob)
+        (all_queues - queues_with_workers).each do |queue|
+          logger.warn "No worker supplied for '#{queue}'"
         end
       end
 
@@ -285,15 +288,17 @@ module Shoryuken
     end
 
     def patch_deprecated_workers!
-      Shoryuken.workers.each do |queue, worker_class|
-        if worker_class.instance_method(:perform).arity == 1
-          logger.warn "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)"
+      Shoryuken.worker_registry.queues.each do |queue|
+        Shoryuken.worker_registry.workers(queue).each do |worker_class|
+          if worker_class.instance_method(:perform).arity == 1
+            logger.warn "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)"
 
-          worker_class.class_eval do
-            alias_method :deprecated_perform, :perform
+            worker_class.class_eval do
+              alias_method :deprecated_perform, :perform
 
-            def perform(sqs_msg, body = nil)
-              deprecated_perform(sqs_msg)
+              def perform(sqs_msg, body = nil)
+                deprecated_perform(sqs_msg)
+              end
             end
           end
         end

data/lib/shoryuken/default_worker_registry.rb

@@ -0,0 +1,46 @@
+module Shoryuken
+  class DefaultWorkerRegistry < WorkerRegistry
+    def initialize
+      @workers = {}
+    end
+
+    def batch_receive_messages?(queue)
+      !!(@workers[queue] && @workers[queue].get_shoryuken_options['batch'])
+    end
+
+    def clear
+      @workers.clear
+    end
+
+    def fetch_worker(queue, message)
+      worker_class = !message.is_a?(Array) &&
+        message.message_attributes &&
+        message.message_attributes['shoryuken_class'] &&
+        message.message_attributes['shoryuken_class'][:string_value]
+
+      worker_class = (worker_class.constantize rescue nil) || @workers[queue]
+
+      worker_class.new
+    end
+
+    def queues
+      @workers.keys
+    end
+
+    def register_worker(queue, clazz)
+      if worker_class = @workers[queue]
+        if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
+          raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+            "because #{worker_class} is already registered for this queue, "\
+            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers."
+        end
+      end
+
+      @workers[queue] = clazz
+    end
+
+    def workers(queue)
+      [@workers.fetch(queue, [])].flatten
+    end
+  end
+end

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -0,0 +1,63 @@
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+
+require 'shoryuken'
+
+module ActiveJob
+  module QueueAdapters
+    # == Shoryuken adapter for Active Job
+    #
+    # Shoryuken ("sho-ryu-ken") is a super-efficient AWS SQS thread based message processor.
+    #
+    # Read more about Shoryuken {here}[https://github.com/phstc/shoryuken].
+    #
+    # To use Shoryuken set the queue_adapter config to +:shoryuken+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :shoryuken
+    class ShoryukenAdapter
+      class << self
+        def enqueue(job) #:nodoc:
+          register_worker!(job)
+
+          Shoryuken::Client.send_message(job.queue_name, job.serialize, message_attributes: message_attributes)
+        end
+
+        def enqueue_at(job, timestamp) #:nodoc:
+          register_worker!(job)
+
+          delay = (timestamp - Time.current.to_f).round
+          raise 'The maximum allowed delay is 15 minutes' if delay > 15.minutes
+
+          Shoryuken::Client.send_message(job.queue_name, job.serialize, delay_seconds: delay,
+                                                                        message_attributes: message_attributes)
+        end
+
+
+        private
+
+        def register_worker!(job)
+          Shoryuken.register_worker(job.queue_name, JobWrapper)
+        end
+
+        def message_attributes
+          @message_attributes ||= {
+            'shoryuken_class' => {
+              string_value: JobWrapper.to_s,
+              data_type: 'String'
+            }
+          }
+        end
+      end
+
+      class JobWrapper #:nodoc:
+        include Shoryuken::Worker
+
+        shoryuken_options body_parser: :json, auto_delete: true
+
+        def perform(sqs_msg, hash)
+          Base.execute hash
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/fetcher.rb

@@ -13,7 +13,7 @@ module Shoryuken
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = Shoryuken.options[:aws][:receive_message].to_h
+      options = Shoryuken.options[:aws][:receive_message].to_h.dup
       options[:limit] = limit
       options[:message_attribute_names] ||= []
       options[:message_attribute_names] << 'shoryuken_class'
@@ -28,8 +28,7 @@ module Shoryuken
         logger.debug "Looking for new messages in '#{queue}'"
 
         begin
-          batch = !!(Shoryuken.workers[queue] && Shoryuken.workers[queue].get_shoryuken_options['batch'])
-
+          batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
           limit = batch ? FETCH_LIMIT : available_processors
 
           if (sqs_msgs = Array(receive_message(queue, limit))).any?

data/lib/shoryuken/launcher.rb

@@ -8,7 +8,8 @@ module Shoryuken
     attr_accessor :manager
 
     def initialize
-      @manager = Shoryuken::Manager.new_link
+      @condvar = Celluloid::Condition.new
+      @manager = Shoryuken::Manager.new_link(@condvar)
       @fetcher = Shoryuken::Fetcher.new_link(manager)
 
       @done = false
@@ -22,7 +23,8 @@ module Shoryuken
         @fetcher.terminate if @fetcher.alive?
 
         manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
-        manager.wait(:shutdown)
+        @condvar.wait
+        manager.terminate
       end
     end
 

data/lib/shoryuken/manager.rb

@@ -10,14 +10,16 @@ module Shoryuken
 
     trap_exit :processor_died
 
-    def initialize
+    def initialize(condvar)
       @count  = Shoryuken.options[:concurrency] || 25
       @queues = Shoryuken.queues.dup.uniq
+      @finished = condvar
 
       @done = false
 
       @busy  = []
-      @ready = @count.times.map { Processor.new_link(current_actor) }
+      @ready = @count.times.map { build_processor }
+      @threads = {}
     end
 
     def start
@@ -39,7 +41,7 @@ module Shoryuken
         end
         @ready.clear
 
-        return after(0) { signal(:shutdown) } if @busy.empty?
+        return after(0) { @finished.signal } if @busy.empty?
 
         if options[:shutdown]
           hard_shutdown_in(options[:timeout])
@@ -53,6 +55,7 @@ module Shoryuken
       watchdog('Manager#processor_done died') do
         logger.info "Process done for '#{queue}'"
 
+        @threads.delete(processor.object_id)
         @busy.delete processor
 
         if stopped?
@@ -65,12 +68,13 @@ module Shoryuken
 
     def processor_died(processor, reason)
       watchdog("Manager#processor_died died") do
-        logger.info "Process died, reason: #{reason}"
+        logger.error "Process died, reason: #{reason}" unless reason.to_s.empty?
 
+        @threads.delete(processor.object_id)
         @busy.delete processor
 
         unless stopped?
-          @ready << Processor.new_link(current_actor)
+          @ready << build_processor
         end
       end
     end
@@ -133,8 +137,18 @@ module Shoryuken
       end
     end
 
+    def real_thread(proxy_id, thr)
+      @threads[proxy_id] = thr
+    end
+
     private
 
+    def build_processor
+      processor = Processor.new_link(current_actor)
+      processor.proxy_id = processor.object_id
+      processor
+    end
+
     def restart_queue!(queue)
       return if stopped?
 
@@ -148,7 +162,6 @@ module Shoryuken
 
           @fetcher_paused = false
 
-
           dispatch
         end
       end
@@ -172,11 +185,10 @@ module Shoryuken
       # get/remove the first queue in the list
       queue = @queues.shift
 
-
-      unless Shoryuken.workers.include? queue
+      unless defined?(::ActiveJob) ||  !Shoryuken.worker_registry.workers(queue).empty?
         # when no worker registered pause the queue to avoid endless recursion
 
-        logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because of no workers registered"
+        logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered"
 
         after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
 
@@ -195,7 +207,7 @@ module Shoryuken
       if @busy.size > 0
         after(delay) { soft_shutdown(delay) }
       else
-        after(0) { signal(:shutdown) }
+        @finished.signal
       end
     end
 
@@ -209,12 +221,13 @@ module Shoryuken
             logger.info { "Hard shutting down #{@busy.size} busy workers" }
 
             @busy.each do |processor|
-              t = processor.bare_object.actual_work_thread
-              t.raise Shutdown if processor.alive?
+              if processor.alive? && t = @threads.delete(processor.object_id)
+                t.raise Shutdown
+              end
             end
           end
 
-          after(0) { signal(:shutdown) }
+          @finished.signal
         end
       end
     end