shoryuken 2.1.3 → 3.0.0

data/bin/shoryuken

@@ -1,12 +1,50 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/shoryuken/cli'
-
-begin
-  Shoryuken::CLI.instance.run(ARGV)
-rescue => e
-  raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
-  exit 1
+require 'rubygems'
+
+require 'thor'
+require 'aws-sdk-core'
+require_relative 'cli/base'
+require_relative 'cli/sqs'
+require_relative '../lib/shoryuken/runner'
+
+# rubocop:disable Metrics/AbcSize
+module Shoryuken
+  module CLI
+    class Runner < Base
+      default_task :start
+
+      register(Shoryuken::CLI::SQS, 'sqs', 'sqs COMMAND', 'SQS commands')
+
+      desc 'start', 'Start shoryuken'
+      method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
+      method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
+      method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
+      method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
+      method_option :config_file, aliases: '-C', type: :string,  desc: 'Path to config file'
+      method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
+      method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
+      method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
+      method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
+      def start
+        opts = options.to_h.symbolize_keys
+
+        # Keep compatibility with old CLI queue format
+        opts[:queues] = options.queues.map { |q| q.split(',') } if options.queues
+
+        if options.daemon && options.logfile.nil?
+          fail_task "You should set a logfile if you're going to daemonize"
+        end
+
+        Shoryuken::Runner.instance.run(opts.freeze)
+      end
+
+      desc 'version', 'Print version'
+      def version
+        say "Shoryuken #{Shoryuken::VERSION}"
+      end
+    end
+  end
 end
+
+Shoryuken::CLI::Runner.start

data/examples/default_worker.rb

@@ -4,6 +4,6 @@ class DefaultWorker
   shoryuken_options queue: 'default', auto_delete: true
 
   def perform(sqs_msg, body)
-    Shoryuken.logger.info("Received message: '#{body}'")
+    Shoryuken.logger.debug("Received message: '#{body}'")
   end
 end

data/lib/shoryuken.rb

@@ -1,12 +1,13 @@
 require 'yaml'
+require 'json'
 require 'aws-sdk-core'
 require 'time'
+require 'concurrent'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
 require 'shoryuken/util'
 require 'shoryuken/logging'
-require 'shoryuken/aws_config'
 require 'shoryuken/environment_loader'
 require 'shoryuken/queue'
 require 'shoryuken/message'
@@ -19,9 +20,11 @@ require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
-require 'shoryuken/sns_arn'
-require 'shoryuken/topic'
 require 'shoryuken/polling'
+require 'shoryuken/manager'
+require 'shoryuken/launcher'
+require 'shoryuken/processor'
+require 'shoryuken/fetcher'
 
 module Shoryuken
   DEFAULTS = {
@@ -33,78 +36,106 @@ module Shoryuken
     lifecycle_events: {
       startup: [],
       quiet: [],
-      shutdown: [],
+      shutdown: []
     },
-    polling_strategy: Polling::WeightedRoundRobin,
-  }
+    polling_strategy: Polling::WeightedRoundRobin
+  }.freeze
 
-  @@queues = []
-  @@worker_registry = DefaultWorkerRegistry.new
+  @@queues                          = []
+  @@worker_registry                 = DefaultWorkerRegistry.new
   @@active_job_queue_name_prefixing = false
+  @@sqs_client                      = nil
+  @@sqs_client_receive_message_opts = {}
+  @@start_callback                  = nil
+  @@stop_callback                   = nil
 
   class << self
-    def options
-      @options ||= DEFAULTS.dup
-    end
-
     def queues
       @@queues
     end
 
-    def logger
-      Shoryuken::Logging.logger
+    def add_queue(queue, priority = 1)
+      priority.times { queues << queue }
     end
 
-    def register_worker(*args)
-      worker_registry.register_worker(*args)
+    def worker_registry
+      @@worker_registry
     end
 
     def worker_registry=(worker_registry)
       @@worker_registry = worker_registry
     end
 
-    def worker_registry
-      @@worker_registry
+    def start_callback
+      @@start_callback
+    end
+
+    def start_callback=(start_callback)
+      @@start_callback = start_callback
+    end
+
+    def stop_callback
+      @@stop_callback
+    end
+
+    def stop_callback=(stop_callback)
+      @@stop_callback = stop_callback
     end
 
     def active_job_queue_name_prefixing
       @@active_job_queue_name_prefixing
     end
 
-    def active_job_queue_name_prefixing=(prefixing)
-      @@active_job_queue_name_prefixing = prefixing
+    def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
+      @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
+    end
+
+    def sqs_client
+      @@sqs_client ||= Aws::SQS::Client.new
+    end
+
+    def sqs_client=(sqs_client)
+      @@sqs_client
+    end
+
+    def sqs_client_receive_message_opts
+      @@sqs_client_receive_message_opts
+    end
+
+    def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+      @@sqs_client_receive_message_opts
+    end
+
+    def options
+      @@options ||= DEFAULTS.dup
+    end
+
+    def logger
+      Shoryuken::Logging.logger
+    end
+
+    def register_worker(*args)
+      @@worker_registry.register_worker(*args)
     end
 
-    ##
-    # Configuration for Shoryuken server, use like:
-    #
-    #   Shoryuken.configure_server do |config|
-    #     config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
-    #   end
     def configure_server
       yield self if server?
     end
 
     def server_middleware
-      @server_chain ||= default_server_middleware
-      yield @server_chain if block_given?
-      @server_chain
+      @@server_chain ||= default_server_middleware
+      yield @@server_chain if block_given?
+      @@server_chain
     end
 
-    ##
-    # Configuration for Shoryuken client, use like:
-    #
-    #   Shoryuken.configure_client do |config|
-    #     config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
-    #   end
     def configure_client
       yield self unless server?
     end
 
     def client_middleware
-      @client_chain ||= default_client_middleware
-      yield @client_chain if block_given?
-      @client_chain
+      @@client_chain ||= default_client_middleware
+      yield @@client_chain if block_given?
+      @@client_chain
     end
 
     def default_worker_options
@@ -114,27 +145,20 @@ module Shoryuken
         'auto_delete'             => false,
         'auto_visibility_timeout' => false,
         'retry_intervals'         => nil,
-        'batch'                   => false }
+        'batch'                   => false
+      }
     end
 
-    def default_worker_options=(options)
-      @@default_worker_options = options
-    end
-
-    def on_aws_initialization(&block)
-      @aws_initialization_callback = block
+    def default_worker_options=(default_worker_options)
+      @@default_worker_options = default_worker_options
     end
 
     def on_start(&block)
-      @start_callback = block
+      @@start_callback = block
     end
 
     def on_stop(&block)
-      @stop_callback = block
-    end
-
-    def aws=(hash)
-      Shoryuken::AwsConfig.setup(hash)
+      @@stop_callback = block
     end
 
     # Register a block to run at a point in the Shoryuken lifecycle.
@@ -151,10 +175,6 @@ module Shoryuken
       options[:lifecycle_events][event] << block
     end
 
-    attr_reader :aws_initialization_callback,
-                :start_callback,
-                :stop_callback
-
     private
 
     def default_server_middleware

data/lib/shoryuken/client.rb

@@ -1,31 +1,19 @@
 module Shoryuken
   class Client
     @@queues = {}
-    @@topics = {}
 
     class << self
       def queues(name)
         @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
       end
 
-      def sns
-        @sns ||= Shoryuken::AwsConfig.sns
-      end
-
-      def sns_arn
-        @sns_arn ||= SnsArn
-      end
-
       def sqs
-        @sqs ||= Shoryuken::AwsConfig.sqs
+        @@sqs ||= Shoryuken.sqs_client
       end
 
-      def topics(name)
-        @@topics[name.to_s] ||= Topic.new(name, sns)
+      def sqs=(sqs)
+        @@sqs = sqs
       end
-
-      attr_accessor :account_id
-      attr_writer :sns, :sqs, :sqs_resource, :sns_arn
     end
   end
 end

data/lib/shoryuken/default_worker_registry.rb

@@ -14,11 +14,15 @@ module Shoryuken
 
     def fetch_worker(queue, message)
       worker_class = !message.is_a?(Array) &&
-        message.message_attributes &&
-        message.message_attributes['shoryuken_class'] &&
-        message.message_attributes['shoryuken_class'][:string_value]
-
-      worker_class = (worker_class.constantize rescue nil) || @workers[queue]
+                     message.message_attributes &&
+                     message.message_attributes['shoryuken_class'] &&
+                     message.message_attributes['shoryuken_class'][:string_value]
+
+      worker_class = begin
+                       worker_class.constantize
+                     rescue
+                       @workers[queue]
+                     end
 
       worker_class.new
     end

data/lib/shoryuken/environment_loader.rb

@@ -28,10 +28,8 @@ module Shoryuken
       prefix_active_job_queue_names
       parse_queues
       require_workers
-      initialize_aws
       validate_queues
       validate_workers
-      patch_deprecated_workers
     end
 
     private
@@ -49,16 +47,6 @@ module Shoryuken
       YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
     end
 
-    # DEPRECATED: Please use configure_server and configure_client in
-    # https://github.com/phstc/shoryuken/blob/a81637d577b36c5cf245882733ea91a335b6602f/lib/shoryuken.rb#L82
-    # Please delete this method afert next release (v2.0.12 or later)
-    def initialize_aws
-      unless Shoryuken.options[:aws].to_h.empty?
-        Shoryuken.logger.warn { '[DEPRECATION] aws in shoryuken.yml is deprecated. Please use configure_server and configure_client in your initializer' }
-      end
-      Shoryuken::AwsConfig.setup(Shoryuken.options[:aws])
-    end
-
     def initialize_logger
       Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
@@ -83,12 +71,11 @@ module Shoryuken
     end
 
     def merge_cli_defined_queues
-      cli_defined_queues = options.delete(:queues) || []
+      cli_defined_queues = options[:queues].to_a
 
       cli_defined_queues.each do |cli_defined_queue|
-        Shoryuken.options[:queues].delete_if do |config_file_queue|
-          config_file_queue[0] == cli_defined_queue[0]
-        end
+        # CLI defined queues override config_file defined queues
+        Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
 
         Shoryuken.options[:queues] << cli_defined_queue
       end
@@ -110,7 +97,7 @@ module Shoryuken
     end
 
     def parse_queue(queue, weight = nil)
-      [weight.to_i, 1].max.times { Shoryuken.queues << queue }
+      Shoryuken.add_queue(queue, [weight.to_i, 1].max)
     end
 
     def parse_queues
@@ -119,24 +106,6 @@ module Shoryuken
       end
     end
 
-    def patch_deprecated_workers
-      Shoryuken.worker_registry.queues.each do |queue|
-        Shoryuken.worker_registry.workers(queue).each do |worker_class|
-          if worker_class.instance_method(:perform).arity == 1
-            Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
-
-            worker_class.class_eval do
-              alias_method :deprecated_perform, :perform
-
-              def perform(sqs_msg, body = nil)
-                deprecated_perform(sqs_msg)
-              end
-            end
-          end
-        end
-      end
-    end
-
     def require_workers
       required = Shoryuken.options[:require]
 
@@ -156,7 +125,7 @@ module Shoryuken
 
       Shoryuken.queues.uniq.each do |queue|
         begin
-          Shoryuken::Client.queues queue
+          Shoryuken::Client.queues(queue)
         rescue Aws::SQS::Errors::NonExistentQueue
           non_existent_queues << queue
         end
@@ -166,13 +135,13 @@ module Shoryuken
     end
 
     def validate_workers
+      return if defined?(::ActiveJob)
+
       all_queues = Shoryuken.queues
       queues_with_workers = Shoryuken.worker_registry.queues
 
-      unless defined?(::ActiveJob)
-        (all_queues - queues_with_workers).each do |queue|
-          Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
-        end
+      (all_queues - queues_with_workers).each do |queue|
+        Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
       end
     end
   end

data/lib/shoryuken/fetcher.rb

@@ -5,23 +5,21 @@ module Shoryuken
     FETCH_LIMIT = 10
 
     def fetch(queue, available_processors)
-      watchdog('Fetcher#fetch died') do
-        started_at = Time.now
-
-        logger.debug { "Looking for new messages in '#{queue}'" }
-
-        begin
-          limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
-
-          sqs_msgs = Array(receive_messages(queue, limit))
-          logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } if !sqs_msgs.empty?
-          logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
-          sqs_msgs
-        rescue => ex
-          logger.error { "Error fetching message: #{ex}" }
-          logger.error { ex.backtrace.first }
-          []
-        end
+      started_at = Time.now
+
+      logger.debug { "Looking for new messages in '#{queue}'" }
+
+      begin
+        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
+
+        sqs_msgs = Array(receive_messages(queue, limit))
+        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
+        sqs_msgs
+      rescue => ex
+        logger.error { "Error fetching message: #{ex}" }
+        logger.error { ex.backtrace.first }
+        []
       end
     end
 
@@ -31,7 +29,7 @@ module Shoryuken
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = (Shoryuken.options[:aws][:receive_message] || {}).dup
+      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
       options[:max_number_of_messages] = limit
       options[:message_attribute_names] = %w(All)
       options[:attribute_names] = %w(All)