shoryuken 2.0.11 → 3.0.6

data/bin/shoryuken

@@ -1,12 +1,58 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/shoryuken/cli'
-
-begin
-  Shoryuken::CLI.instance.run(ARGV)
-rescue => e
-  raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
-  exit 1
+require 'rubygems'
+
+require 'thor'
+require 'aws-sdk-core'
+require_relative 'cli/base'
+require_relative 'cli/sqs'
+require_relative '../lib/shoryuken/runner'
+
+# rubocop:disable Metrics/AbcSize
+module Shoryuken
+  module CLI
+    class Runner < Base
+      default_task :start
+
+      register(Shoryuken::CLI::SQS, 'sqs', 'sqs COMMAND', 'SQS commands')
+
+      desc 'start', 'Starts shoryuken'
+      method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
+      method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
+      method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
+      method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
+      method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
+      method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
+      method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
+      method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
+      method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
+      method_option :delay,       aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
+      def start
+        opts = options.to_h.symbolize_keys
+
+        if opts[:config_file]
+          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+        end
+
+        opts[:config_file] = opts.delete(:config) if opts[:config]
+
+        # Keep compatibility with old CLI queue format
+        opts[:queues] = options[:queues].map { |q| q.split(',') } if options[:queues]
+
+        if options[:daemon] && options[:logfile].nil?
+          fail_task "You should set a logfile if you're going to daemonize"
+        end
+
+        Shoryuken::Runner.instance.run(opts.freeze)
+      end
+
+      desc 'version', 'Prints version'
+      def version
+        say "Shoryuken #{Shoryuken::VERSION}"
+      end
+    end
+  end
 end
+
+Shoryuken::CLI::Runner.start

data/examples/default_worker.rb

@@ -4,17 +4,6 @@ class DefaultWorker
   shoryuken_options queue: 'default', auto_delete: true
 
   def perform(sqs_msg, body)
-    puts "DefaultWorker: '#{body}'"
-  end
-end
-
-# multiple workers for the same queue
-class DefaultWorker2
-  include Shoryuken::Worker
-
-  shoryuken_options queue: 'default', auto_delete: true
-
-  def perform(sqs_msg, body)
-    puts "DefaultWorker2: '#{body}'"
+    Shoryuken.logger.debug("Received message: '#{body}'")
   end
 end

data/lib/shoryuken/client.rb

@@ -1,40 +1,18 @@
 module Shoryuken
   class Client
     @@queues = {}
-    @@topics = {}
 
     class << self
       def queues(name)
         @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
       end
 
-      def sns
-        @sns ||= Aws::SNS::Client.new(aws_client_options(:sns_endpoint))
-      end
-
-      def sns_arn
-        @sns_arn ||= SnsArn
-      end
-
       def sqs
-        @sqs ||= Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
+        @@sqs ||= Shoryuken.sqs_client
       end
 
-      def topics(name)
-        @@topics[name.to_s] ||= Topic.new(name, sns)
-      end
-
-      attr_accessor :account_id
-      attr_writer :sns, :sqs, :sqs_resource, :sns_arn
-
-      private
-
-      def aws_client_options(service_endpoint_key)
-        environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
-        explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key] || environment_endpoint
-        options = {}
-        options[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
-        options
+      def sqs=(sqs)
+        @@sqs = sqs
       end
     end
   end

data/lib/shoryuken/default_worker_registry.rb

@@ -14,11 +14,15 @@ module Shoryuken
 
     def fetch_worker(queue, message)
       worker_class = !message.is_a?(Array) &&
-        message.message_attributes &&
-        message.message_attributes['shoryuken_class'] &&
-        message.message_attributes['shoryuken_class'][:string_value]
-
-      worker_class = (worker_class.constantize rescue nil) || @workers[queue]
+                     message.message_attributes &&
+                     message.message_attributes['shoryuken_class'] &&
+                     message.message_attributes['shoryuken_class'][:string_value]
+
+      worker_class = begin
+                       worker_class.constantize
+                     rescue
+                       @workers[queue]
+                     end
 
       worker_class.new
     end

data/lib/shoryuken/environment_loader.rb

@@ -2,30 +2,34 @@ module Shoryuken
   class EnvironmentLoader
     attr_reader :options
 
-    def self.load(options)
-      new(options).load
+    def self.setup_options(options)
+      instance = new(options)
+      instance.setup_options
+      instance
     end
 
     def self.load_for_rails_console
-      load(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
+      instance = setup_options(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
+      instance.load
     end
 
     def initialize(options)
       @options = options
     end
 
-    def load
-      load_rails if options[:rails]
+    def setup_options
       initialize_options
       initialize_logger
       merge_cli_defined_queues
+    end
+
+    def load
+      load_rails if options[:rails]
       prefix_active_job_queue_names
       parse_queues
       require_workers
-      initialize_aws
       validate_queues
       validate_workers
-      patch_deprecated_workers
     end
 
     private
@@ -43,40 +47,9 @@ module Shoryuken
       YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
     end
 
-    def initialize_aws
-      # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-      # when not explicit supplied
-      return if Shoryuken.options[:aws].empty?
-
-      shoryuken_keys = %w(
-        account_id
-        sns_endpoint
-        sqs_endpoint
-        receive_message).map(&:to_sym)
-
-      aws_options = Shoryuken.options[:aws].reject do |k, v|
-        shoryuken_keys.include?(k)
-      end
-
-      # assume credentials based authentication
-      credentials = Aws::Credentials.new(
-        aws_options.delete(:access_key_id),
-        aws_options.delete(:secret_access_key))
-
-      # but only if the configuration options have valid values
-      aws_options = aws_options.merge(credentials: credentials) if credentials.set?
-
-      if (callback = Shoryuken.aws_initialization_callback)
-        Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
-        callback.call(aws_options)
-      end
-
-      Aws.config = aws_options
-    end
-
     def initialize_logger
-      Shoryuken::Logging.initialize_logger(options[:logfile]) if options[:logfile]
-      Shoryuken.logger.level = Logger::DEBUG if options[:verbose]
+      Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
+      Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
     def load_rails
@@ -98,12 +71,11 @@ module Shoryuken
     end
 
     def merge_cli_defined_queues
-      cli_defined_queues = options.delete(:queues) || []
+      cli_defined_queues = options[:queues].to_a
 
       cli_defined_queues.each do |cli_defined_queue|
-        Shoryuken.options[:queues].delete_if do |config_file_queue|
-          config_file_queue[0] == cli_defined_queue[0]
-        end
+        # CLI defined queues override config_file defined queues
+        Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
 
         Shoryuken.options[:queues] << cli_defined_queue
       end
@@ -125,7 +97,7 @@ module Shoryuken
     end
 
     def parse_queue(queue, weight = nil)
-      [weight.to_i, 1].max.times { Shoryuken.queues << queue }
+      Shoryuken.add_queue(queue, [weight.to_i, 1].max)
     end
 
     def parse_queues
@@ -134,28 +106,18 @@ module Shoryuken
       end
     end
 
-    def patch_deprecated_workers
-      Shoryuken.worker_registry.queues.each do |queue|
-        Shoryuken.worker_registry.workers(queue).each do |worker_class|
-          if worker_class.instance_method(:perform).arity == 1
-            Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
+    def require_workers
+      required = Shoryuken.options[:require]
 
-            worker_class.class_eval do
-              alias_method :deprecated_perform, :perform
+      return unless required
 
-              def perform(sqs_msg, body = nil)
-                deprecated_perform(sqs_msg)
-              end
-            end
-          end
-        end
+      if File.directory?(required)
+        Dir[File.join(required, '**', '*.rb')].each(&method(:require))
+      else
+        require required
       end
     end
 
-    def require_workers
-      require Shoryuken.options[:require] if Shoryuken.options[:require]
-    end
-
     def validate_queues
       Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
 
@@ -163,7 +125,7 @@ module Shoryuken
 
       Shoryuken.queues.uniq.each do |queue|
         begin
-          Shoryuken::Client.queues queue
+          Shoryuken::Client.queues(queue)
         rescue Aws::SQS::Errors::NonExistentQueue
           non_existent_queues << queue
         end
@@ -173,13 +135,13 @@ module Shoryuken
     end
 
     def validate_workers
+      return if defined?(::ActiveJob)
+
       all_queues = Shoryuken.queues
       queues_with_workers = Shoryuken.worker_registry.queues
 
-      unless defined?(::ActiveJob)
-        (all_queues - queues_with_workers).each do |queue|
-          Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
-        end
+      (all_queues - queues_with_workers).each do |queue|
+        Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
       end
     end
   end

data/lib/shoryuken/fetcher.rb

@@ -1,73 +1,42 @@
 module Shoryuken
   class Fetcher
-    include Celluloid
     include Util
 
     FETCH_LIMIT = 10
 
-    def initialize(manager)
-      @manager = manager
+    def fetch(queue, available_processors)
+      started_at = Time.now
+
+      logger.debug { "Looking for new messages in '#{queue}'" }
+
+      begin
+        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
+
+        sqs_msgs = Array(receive_messages(queue, limit))
+        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
+        sqs_msgs
+      rescue => ex
+        logger.error { "Error fetching message: #{ex}" }
+        logger.error { ex.backtrace.first }
+        []
+      end
     end
 
+    private
+
     def receive_messages(queue, limit)
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = (Shoryuken.options[:aws][:receive_message] || {}).dup
+      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
       options[:max_number_of_messages] = limit
       options[:message_attribute_names] = %w(All)
       options[:attribute_names] = %w(All)
 
-      Shoryuken::Client.queues(queue).receive_messages options
-    end
-
-    def fetch(queue, available_processors)
-      watchdog('Fetcher#fetch died') do
-        started_at = Time.now
-
-        logger.debug { "Looking for new messages in '#{queue}'" }
-
-        begin
-          batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
-          limit = batch ? FETCH_LIMIT : available_processors
-
-          if (sqs_msgs = Array(receive_messages(queue, limit))).any?
-            logger.info { "Found #{sqs_msgs.size} messages for '#{queue}'" }
-
-            if batch
-              @manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
-            else
-              sqs_msgs.each { |sqs_msg| @manager.async.assign(queue, sqs_msg) }
-            end
-
-            @manager.async.rebalance_queue_weight!(queue)
-          else
-            logger.debug { "No message found for '#{queue}'" }
-
-            @manager.async.pause_queue!(queue)
-          end
-
-          logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
-        rescue => ex
-          logger.error { "Error fetching message: #{ex}" }
-          logger.error { ex.backtrace.first }
-        end
-
-        @manager.async.dispatch
-      end
-
-    end
-
-    private
-
-    def patch_sqs_msgs!(sqs_msgs)
-      sqs_msgs.instance_eval do
-        def message_id
-          "batch-with-#{size}-messages"
-        end
-      end
+      options.merge!(queue.options)
 
-      sqs_msgs
+      Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -1,43 +1,19 @@
 module Shoryuken
   class Launcher
-    include Celluloid
     include Util
 
-    trap_exit :actor_died
-
-    attr_accessor :manager
-
     def initialize
-      @condvar = Celluloid::Condition.new
-      @manager = Shoryuken::Manager.new_link(@condvar)
-      @fetcher = Shoryuken::Fetcher.new_link(manager)
-
-      @done = false
-
-      manager.fetcher = @fetcher
+      @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
+                                        Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
     end
 
     def stop(options = {})
-      watchdog('Launcher#stop') do
-        @done = true
-        @fetcher.terminate if @fetcher.alive?
-
-        manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
-        @condvar.wait
-        manager.terminate
-      end
+      @manager.stop(shutdown: !options[:shutdown].nil?,
+                    timeout: Shoryuken.options[:timeout])
     end
 
     def run
-      watchdog('Launcher#run') do
-        manager.async.start
-      end
-    end
-
-    def actor_died(actor, reason)
-      return if @done
-      logger.warn { 'Shoryuken died due to the following error, cannot recover, process exiting' }
-      exit 1
+      @manager.start
     end
   end
 end