shoryuken 2.1.1 → 3.1.3

data/lib/shoryuken/util.rb

@@ -1,13 +1,5 @@
 module Shoryuken
   module Util
-    def watchdog(last_words)
-      yield
-    rescue => ex
-      logger.error { last_words }
-      logger.error { ex }
-      logger.error { ex.backtrace.join("\n") }
-    end
-
     def logger
       Shoryuken.logger
     end
@@ -38,7 +30,7 @@ module Shoryuken
     end
 
     def worker_name(worker_class, sqs_msg, body = nil)
-      if defined?(::ActiveJob) \
+      if Shoryuken.active_job? \
           && !sqs_msg.is_a?(Array) \
           && sqs_msg.message_attributes \
           && sqs_msg.message_attributes['shoryuken_class'] \

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '2.1.1'
+  VERSION = '3.1.3'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -48,6 +48,14 @@ module Shoryuken
         !!get_shoryuken_options['auto_visibility_timeout']
       end
 
+      def exponential_backoff?
+        !!get_shoryuken_options['retry_intervals']
+      end
+
+      def auto_delete?
+        !!(get_shoryuken_options['delete'] || get_shoryuken_options['auto_delete'])
+      end
+
       def get_shoryuken_options # :nodoc:
         @shoryuken_options || Shoryuken.default_worker_options
       end
@@ -63,7 +71,7 @@ module Shoryuken
 
       def normalize_worker_queue!
         queue = @shoryuken_options['queue']
-        if queue.respond_to? :call
+        if queue.respond_to?(:call)
           queue = queue.call
           @shoryuken_options['queue'] = queue
         end

data/lib/shoryuken.rb

@@ -1,12 +1,13 @@
 require 'yaml'
+require 'json'
 require 'aws-sdk-core'
 require 'time'
+require 'concurrent'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
 require 'shoryuken/util'
 require 'shoryuken/logging'
-require 'shoryuken/aws_config'
 require 'shoryuken/environment_loader'
 require 'shoryuken/queue'
 require 'shoryuken/message'
@@ -19,163 +20,52 @@ require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
-require 'shoryuken/sns_arn'
-require 'shoryuken/topic'
+require 'shoryuken/polling/base'
+require 'shoryuken/polling/weighted_round_robin'
+require 'shoryuken/polling/strict_priority'
+require 'shoryuken/manager'
+require 'shoryuken/launcher'
+require 'shoryuken/processor'
+require 'shoryuken/fetcher'
+require 'shoryuken/options'
 
 module Shoryuken
-  DEFAULTS = {
-    concurrency: 25,
-    queues: [],
-    aws: {},
-    delay: 0,
-    timeout: 8,
-    lifecycle_events: {
-      startup: [],
-      quiet: [],
-      shutdown: [],
-    }
-  }
-
-  @@queues = []
-  @@worker_registry = DefaultWorkerRegistry.new
-  @@active_job_queue_name_prefixing = false
-
-  class << self
-    def options
-      @options ||= DEFAULTS.dup
-    end
-
-    def queues
-      @@queues
-    end
-
-    def logger
-      Shoryuken::Logging.logger
-    end
-
-    def register_worker(*args)
-      worker_registry.register_worker(*args)
-    end
-
-    def worker_registry=(worker_registry)
-      @@worker_registry = worker_registry
-    end
-
-    def worker_registry
-      @@worker_registry
-    end
-
-    def active_job_queue_name_prefixing
-      @@active_job_queue_name_prefixing
-    end
-
-    def active_job_queue_name_prefixing=(prefixing)
-      @@active_job_queue_name_prefixing = prefixing
-    end
-
-    ##
-    # Configuration for Shoryuken server, use like:
-    #
-    #   Shoryuken.configure_server do |config|
-    #     config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
-    #   end
-    def configure_server
-      yield self if server?
-    end
-
-    def server_middleware
-      @server_chain ||= default_server_middleware
-      yield @server_chain if block_given?
-      @server_chain
-    end
-
-    ##
-    # Configuration for Shoryuken client, use like:
-    #
-    #   Shoryuken.configure_client do |config|
-    #     config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
-    #   end
-    def configure_client
-      yield self unless server?
-    end
-
-    def client_middleware
-      @client_chain ||= default_client_middleware
-      yield @client_chain if block_given?
-      @client_chain
-    end
-
-    def default_worker_options
-      @@default_worker_options ||= {
-        'queue'                   => 'default',
-        'delete'                  => false,
-        'auto_delete'             => false,
-        'auto_visibility_timeout' => false,
-        'retry_intervals'         => nil,
-        'batch'                   => false }
-    end
-
-    def default_worker_options=(options)
-      @@default_worker_options = options
-    end
-
-    def on_aws_initialization(&block)
-      @aws_initialization_callback = block
-    end
-
-    def on_start(&block)
-      @start_callback = block
-    end
-
-    def on_stop(&block)
-      @stop_callback = block
-    end
-
-    def aws=(hash)
-      Shoryuken::AwsConfig.setup(hash)
-    end
-
-    # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
-    #
-    #   Shoryuken.configure_server do |config|
-    #     config.on(:shutdown) do
-    #       puts "Goodbye cruel world!"
-    #     end
-    #   end
-    def on(event, &block)
-      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-      options[:lifecycle_events][event] << block
-    end
-
-    attr_reader :aws_initialization_callback,
-                :start_callback,
-                :stop_callback
-
-    private
-
-    def default_server_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Timing
-        m.add Middleware::Server::ExponentialBackoffRetry
-        m.add Middleware::Server::AutoDelete
-        m.add Middleware::Server::AutoExtendVisibility
-        if defined?(::ActiveRecord::Base)
-          require 'shoryuken/middleware/server/active_record'
-          m.add Middleware::Server::ActiveRecord
-        end
-      end
-    end
-
-    def default_client_middleware
-      Middleware::Chain.new
-    end
-
-    def server?
-      defined?(Shoryuken::CLI)
-    end
-  end
+  extend SingleForwardable
+
+  def_delegators(
+    :'Shoryuken::Options',
+    :active_job?,
+    :add_group,
+    :groups,
+    :add_queue,
+    :ungrouped_queues,
+    :worker_registry,
+    :worker_registry=,
+    :polling_strategy,
+    :start_callback,
+    :start_callback=,
+    :stop_callback,
+    :stop_callback=,
+    :active_job_queue_name_prefixing,
+    :active_job_queue_name_prefixing=,
+    :sqs_client,
+    :sqs_client=,
+    :sqs_client_receive_message_opts,
+    :sqs_client_receive_message_opts=,
+    :options,
+    :logger,
+    :register_worker,
+    :configure_server,
+    :server?,
+    :server_middleware,
+    :configure_client,
+    :client_middleware,
+    :default_worker_options,
+    :default_worker_options=,
+    :on_start,
+    :on_stop,
+    :on
+  )
 end
 
-require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
+require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?

data/shoryuken.gemspec

@@ -6,14 +6,14 @@ require 'shoryuken/version'
 Gem::Specification.new do |spec|
   spec.name          = 'shoryuken'
   spec.version       = Shoryuken::VERSION
-  spec.authors       = ['Pablo Cantero', 'Mario Kostelac']
-  spec.email         = ['pablo@pablocantero.com', 'mariokostelac@gmail.com']
-  spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
+  spec.authors       = ['Pablo Cantero']
+  spec.email         = ['pablo@pablocantero.com']
+  spec.description = spec.summary = 'Shoryuken is a super efficient AWS SQS thread based message processor'
   spec.homepage      = 'https://github.com/phstc/shoryuken'
   spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = %w[shoryuken]
+  spec.executables   = %w(shoryuken)
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'nokogiri'
   spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk-core', '~> 2'
-  spec.add_dependency 'celluloid', '~> 0.16'
+  spec.add_dependency 'aws-sdk-core', '> 2'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'thor'
 end

data/spec/integration/launcher_spec.rb

@@ -1,29 +1,36 @@
 require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
+require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
   describe 'Consuming messages', slow: :true do
     before do
-      Shoryuken.options[:aws][:receive_message] = { wait_time_seconds: 5 }
+      Aws.config[:stub_responses] = false
+      Aws.config[:region] = 'us-east-1'
 
       StandardWorker.received_messages = 0
 
-      queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
+      queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
 
-      Shoryuken::Client.sqs.create_queue queue_name: queue
+      Shoryuken::Client.sqs.create_queue(queue_name: queue)
 
-      Shoryuken.queues << queue
+      Shoryuken.add_group('default', 1)
+      Shoryuken.add_queue(queue, 1, 'default')
 
       StandardWorker.get_shoryuken_options['queue'] = queue
 
-      Shoryuken.register_worker queue, StandardWorker
+      Shoryuken.register_worker(queue, StandardWorker)
     end
 
     after do
-      queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: StandardWorker.get_shoryuken_options['queue']).queue_url
+      Aws.config[:stub_responses] = true
 
-      Shoryuken::Client.sqs.delete_queue queue_url: queue_url
+      queue_url = Shoryuken::Client.sqs.get_queue_url(
+        queue_name: StandardWorker.get_shoryuken_options['queue']
+      ).queue_url
+
+      Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
     end
 
     it 'consumes as a command worker' do
@@ -60,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     def poll_queues_until
-      subject.run
+      subject.start
 
       Timeout::timeout(10) do
         begin

data/spec/shoryuken/client_spec.rb

@@ -1,17 +1,16 @@
 require 'spec_helper'
 
-describe Shoryuken::Client do
+RSpec.describe Shoryuken::Client do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs)         { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name)  { 'shoryuken' }
   let(:queue_url)   { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
-  let(:sqs_endpoint) { 'http://localhost:4568' }
-  let(:sns_endpoint) { 'http://0.0.0.0:4568'   }
 
   describe '.queue' do
     before do
       described_class.sqs = sqs
     end
+
     it 'memoizes queues' do
       sqs.stub_responses(:get_queue_url, { queue_url: queue_url }, { queue_url: 'xyz' })
 
@@ -19,46 +18,4 @@ describe Shoryuken::Client do
       expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
     end
   end
-
-  describe 'environment variable endpoints' do
-    before do
-      ENV['AWS_SQS_ENDPOINT'] = sqs_endpoint
-      ENV['AWS_SNS_ENDPOINT'] = sns_endpoint
-      ENV['AWS_REGION'] = 'us-east-1'
-      Shoryuken.options[:aws] = {}
-      Shoryuken::AwsConfig.options = {}
-    end
-
-    it 'will use config file settings if set' do
-      load_config_file_by_file_name('shoryuken_endpoint.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://github.com/phstc/shoryuken:4568')
-      expect(described_class.sns.config.endpoint.to_s).to eq('http://127.0.0.1:4568')
-    end
-
-    it 'should fallback to environment variable if config file not found or set' do
-      load_config_file_by_file_name(nil)
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'should fallback to environment variable if config file found but settings not set' do
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'will fallback to default settings if no config file settings or environment variables found' do
-      ENV['AWS_SQS_ENDPOINT'] = nil
-      ENV['AWS_SNS_ENDPOINT'] = nil
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://sqs.us-east-1.amazonaws.com')
-      expect(described_class.sns.config.endpoint.to_s).to eq('https://sns.us-east-1.amazonaws.com')
-    end
-  end
-
-  def load_config_file_by_file_name(file_name)
-    path_name = file_name ? File.join(File.expand_path('../../..', __FILE__), 'spec', file_name) : nil
-    loader = Shoryuken::EnvironmentLoader.setup_options(config_file: path_name)
-    loader.load
-  end
 end

data/spec/shoryuken/default_worker_registry_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 
-describe Shoryuken::DefaultWorkerRegistry do
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::DefaultWorkerRegistry do
   class RegistryTestWorker
     include Shoryuken::Worker
 
@@ -42,16 +43,17 @@ describe Shoryuken::DefaultWorkerRegistry do
   end
 
   describe 'a registry with workers is handling messages' do
-    def build_message queue, explicit_worker = nil
+    def build_message(queue, explicit_worker = nil)
       attributes = {}
-      attributes['shoryuken_class'] = {
-        string_value: explicit_worker.to_s,
-        data_type: 'String' } if explicit_worker
-
-      double Shoryuken::Message,
-        body: 'test',
-        message_attributes: attributes,
-        message_id: SecureRandom.uuid
+
+      if explicit_worker
+        attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' }
+      end
+
+      double(Shoryuken::Message,
+             body: 'test',
+             message_attributes: attributes,
+             message_id: SecureRandom.uuid)
     end
 
     context 'a batch of messages is being processed' do

data/spec/shoryuken/environment_loader_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'active_job'
+
+RSpec.describe Shoryuken::EnvironmentLoader do
+  subject { described_class.new({}) }
+
+  describe '#parse_queues' do
+    before do
+      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:prefix_active_job_queue_names)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+    end
+
+    specify do
+      Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
+    end
+  end
+
+  describe '#prefix_active_job_queue_names' do
+    before do
+      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+
+      ActiveJob::Base.queue_name_prefix    = 'test'
+      ActiveJob::Base.queue_name_delimiter = '_'
+
+      allow(Shoryuken).to receive(:active_job?).and_return(true)
+    end
+
+    specify do
+      Shoryuken.active_job_queue_name_prefixing = true
+
+      Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
+
+      Shoryuken.options[:groups] = {
+        'group1' => { queues: %w(group1_queue1 group1_queue2) }
+      }
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w(test_queue1 test_queue2 test_queue2))
+      expect(Shoryuken.groups['group1'][:queues]).to eq(%w(test_group1_queue1 test_group1_queue2))
+    end
+  end
+end

data/spec/shoryuken/fetcher_spec.rb

@@ -2,69 +2,50 @@ require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
-describe Shoryuken::Fetcher do
-  let(:manager)    { double Shoryuken::Manager }
-  let(:queue)      { double Shoryuken::Queue }
-  let(:queue_name) { 'default' }
+RSpec.describe Shoryuken::Fetcher do
+  let(:queue)        { instance_double('Shoryuken::Queue') }
+  let(:queue_name)   { 'default' }
+  let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
+  let(:group)        { 'default' }
 
   let(:sqs_msg) do
-    double Shoryuken::Message,
+    double(
+      Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
-      message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
-  end
-
-  subject { described_class.new(manager) }
-
-  before do
-    allow(manager).to receive(:async).and_return(manager)
-    allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+      message_id: 'fc754df79cc24c4196ca5996a44b771e',
+    )
   end
 
+  subject { described_class.new(group) }
 
   describe '#fetch' do
-    it 'calls pause when no message' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).and_return([])
-
-      expect(manager).to receive(:pause_queue!).with(queue_name)
-      expect(manager).to receive(:dispatch)
-
-      subject.fetch(queue_name, 1)
-    end
-
-    it 'assigns messages' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
-
-      expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-      expect(manager).to receive(:assign).with(queue_name, sqs_msg)
-      expect(manager).to receive(:dispatch)
-
-      subject.fetch(queue_name, 5)
-    end
+    let(:limit) { 1 }
 
-    it 'assigns messages in batch' do
-      TestWorker.get_shoryuken_options['batch'] = true
+    specify do
+      expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
 
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
+      Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
 
-      expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-      expect(manager).to receive(:assign).with(queue_name, [sqs_msg])
-      expect(manager).to receive(:dispatch)
+      expect(queue).to receive(:receive_messages).
+        with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
+        and_return([])
 
-      subject.fetch(queue_name, 5)
+      subject.fetch(queue_config, limit)
     end
 
-    context 'when worker not found' do
-      let(:queue_name) { 'notfound' }
+    context 'when limit is greater than FETCH_LIMIT' do
+      let(:limit) { 20 }
 
-      it 'ignores batch' do
-        allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
+      specify do
+        Shoryuken.sqs_client_receive_message_opts[group] = {}
 
-        expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-        expect(manager).to receive(:assign).with(queue_name, sqs_msg)
-        expect(manager).to receive(:dispatch)
+        allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+        expect(queue).to receive(:receive_messages).
+          with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
+          and_return([])
 
-        subject.fetch(queue_name, 5)
+        subject.fetch(queue_config, limit)
       end
     end
   end