shoryuken 2.0.11 → 3.0.0

data/lib/shoryuken.rb

@@ -1,6 +1,8 @@
 require 'yaml'
+require 'json'
 require 'aws-sdk-core'
 require 'time'
+require 'concurrent'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
@@ -18,8 +20,11 @@ require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
-require 'shoryuken/sns_arn'
-require 'shoryuken/topic'
+require 'shoryuken/polling'
+require 'shoryuken/manager'
+require 'shoryuken/launcher'
+require 'shoryuken/processor'
+require 'shoryuken/fetcher'
 
 module Shoryuken
   DEFAULTS = {
@@ -31,45 +36,86 @@ module Shoryuken
     lifecycle_events: {
       startup: [],
       quiet: [],
-      shutdown: [],
-    }
-  }
+      shutdown: []
+    },
+    polling_strategy: Polling::WeightedRoundRobin
+  }.freeze
 
-  @@queues = []
-  @@worker_registry = DefaultWorkerRegistry.new
+  @@queues                          = []
+  @@worker_registry                 = DefaultWorkerRegistry.new
   @@active_job_queue_name_prefixing = false
+  @@sqs_client                      = nil
+  @@sqs_client_receive_message_opts = {}
+  @@start_callback                  = nil
+  @@stop_callback                   = nil
 
   class << self
-    def options
-      @options ||= DEFAULTS.dup
-    end
-
     def queues
       @@queues
     end
 
-    def logger
-      Shoryuken::Logging.logger
+    def add_queue(queue, priority = 1)
+      priority.times { queues << queue }
     end
 
-    def register_worker(*args)
-      worker_registry.register_worker(*args)
+    def worker_registry
+      @@worker_registry
     end
 
     def worker_registry=(worker_registry)
       @@worker_registry = worker_registry
     end
 
-    def worker_registry
-      @@worker_registry
+    def start_callback
+      @@start_callback
+    end
+
+    def start_callback=(start_callback)
+      @@start_callback = start_callback
+    end
+
+    def stop_callback
+      @@stop_callback
+    end
+
+    def stop_callback=(stop_callback)
+      @@stop_callback = stop_callback
     end
 
     def active_job_queue_name_prefixing
       @@active_job_queue_name_prefixing
     end
 
-    def active_job_queue_name_prefixing=(prefixing)
-      @@active_job_queue_name_prefixing = prefixing
+    def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
+      @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
+    end
+
+    def sqs_client
+      @@sqs_client ||= Aws::SQS::Client.new
+    end
+
+    def sqs_client=(sqs_client)
+      @@sqs_client
+    end
+
+    def sqs_client_receive_message_opts
+      @@sqs_client_receive_message_opts
+    end
+
+    def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+      @@sqs_client_receive_message_opts
+    end
+
+    def options
+      @@options ||= DEFAULTS.dup
+    end
+
+    def logger
+      Shoryuken::Logging.logger
+    end
+
+    def register_worker(*args)
+      @@worker_registry.register_worker(*args)
     end
 
     def configure_server
@@ -77,19 +123,19 @@ module Shoryuken
     end
 
     def server_middleware
-      @server_chain ||= default_server_middleware
-      yield @server_chain if block_given?
-      @server_chain
+      @@server_chain ||= default_server_middleware
+      yield @@server_chain if block_given?
+      @@server_chain
     end
 
     def configure_client
-      yield self
+      yield self unless server?
     end
 
     def client_middleware
-      @client_chain ||= default_client_middleware
-      yield @client_chain if block_given?
-      @client_chain
+      @@client_chain ||= default_client_middleware
+      yield @@client_chain if block_given?
+      @@client_chain
     end
 
     def default_worker_options
@@ -99,23 +145,20 @@ module Shoryuken
         'auto_delete'             => false,
         'auto_visibility_timeout' => false,
         'retry_intervals'         => nil,
-        'batch'                   => false }
-    end
-
-    def default_worker_options=(options)
-      @@default_worker_options = options
+        'batch'                   => false
+      }
     end
 
-    def on_aws_initialization(&block)
-      @aws_initialization_callback = block
+    def default_worker_options=(default_worker_options)
+      @@default_worker_options = default_worker_options
     end
 
     def on_start(&block)
-      @start_callback = block
+      @@start_callback = block
     end
 
     def on_stop(&block)
-      @stop_callback = block
+      @@stop_callback = block
     end
 
     # Register a block to run at a point in the Shoryuken lifecycle.
@@ -132,10 +175,6 @@ module Shoryuken
       options[:lifecycle_events][event] << block
     end
 
-    attr_reader :aws_initialization_callback,
-                :start_callback,
-                :stop_callback
-
     private
 
     def default_server_middleware

data/shoryuken.gemspec

@@ -6,14 +6,14 @@ require 'shoryuken/version'
 Gem::Specification.new do |spec|
   spec.name          = 'shoryuken'
   spec.version       = Shoryuken::VERSION
-  spec.authors       = ['Pablo Cantero', 'Mario Kostelac']
-  spec.email         = ['pablo@pablocantero.com', 'mariokostelac@gmail.com']
-  spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
+  spec.authors       = ['Pablo Cantero']
+  spec.email         = ['pablo@pablocantero.com']
+  spec.description = spec.summary = 'Shoryuken is a super efficient AWS SQS thread based message processor'
   spec.homepage      = 'https://github.com/phstc/shoryuken'
   spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = %w[shoryuken]
+  spec.executables   = %w(shoryuken)
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'nokogiri'
   spec.add_development_dependency 'dotenv'
 
   spec.add_dependency 'aws-sdk-core', '~> 2'
-  spec.add_dependency 'celluloid', '~> 0.16'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'thor'
 end

data/spec/integration/launcher_spec.rb

@@ -1,12 +1,11 @@
 require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
+require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
   describe 'Consuming messages', slow: :true do
     before do
-      Shoryuken.options[:aws][:receive_message] = { wait_time_seconds: 5 }
-
       StandardWorker.received_messages = 0
 
       queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
@@ -21,7 +20,9 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     after do
-      queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: StandardWorker.get_shoryuken_options['queue']).queue_url
+      queue_url = Shoryuken::Client.sqs.get_queue_url(
+        queue_name: StandardWorker.get_shoryuken_options['queue']
+      ).queue_url
 
       Shoryuken::Client.sqs.delete_queue queue_url: queue_url
     end

data/spec/shoryuken/client_spec.rb

@@ -1,17 +1,16 @@
 require 'spec_helper'
 
-describe Shoryuken::Client do
+RSpec.describe Shoryuken::Client do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs)         { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name)  { 'shoryuken' }
   let(:queue_url)   { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
-  let(:sqs_endpoint) { 'http://localhost:4568' }
-  let(:sns_endpoint) { 'http://0.0.0.0:4568'   }
 
   describe '.queue' do
     before do
       described_class.sqs = sqs
     end
+
     it 'memoizes queues' do
       sqs.stub_responses(:get_queue_url, { queue_url: queue_url }, { queue_url: 'xyz' })
 
@@ -19,44 +18,4 @@ describe Shoryuken::Client do
       expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
     end
   end
-
-  describe 'environment variable endpoints' do
-    before do
-      ENV['AWS_SQS_ENDPOINT'] = sqs_endpoint
-      ENV['AWS_SNS_ENDPOINT'] = sns_endpoint
-      ENV['AWS_REGION'] = 'us-east-1'
-      Shoryuken.options[:aws] = {}
-    end
-
-    it 'will use config file settings if set' do
-      load_config_file_by_file_name('shoryuken_endpoint.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://github.com/phstc/shoryuken:4568')
-      expect(described_class.sns.config.endpoint.to_s).to eq('http://127.0.0.1:4568')
-    end
-
-    it 'should fallback to environment variable if config file not found or set' do
-      load_config_file_by_file_name(nil)
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'should fallback to environment variable if config file found but settings not set' do
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'will fallback to default settings if no config file settings or environment variables found' do
-      ENV['AWS_SQS_ENDPOINT'] = nil
-      ENV['AWS_SNS_ENDPOINT'] = nil
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://sqs.us-east-1.amazonaws.com')
-      expect(described_class.sns.config.endpoint.to_s).to eq('https://sns.us-east-1.amazonaws.com')
-    end
-  end
-
-  def load_config_file_by_file_name(file_name)
-    path_name = file_name ? File.join(File.expand_path('../../..', __FILE__), 'spec', file_name) : nil
-    Shoryuken::EnvironmentLoader.load(config_file: path_name)
-  end
 end

data/spec/shoryuken/default_worker_registry_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 
-describe Shoryuken::DefaultWorkerRegistry do
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::DefaultWorkerRegistry do
   class RegistryTestWorker
     include Shoryuken::Worker
 
@@ -42,16 +43,17 @@ describe Shoryuken::DefaultWorkerRegistry do
   end
 
   describe 'a registry with workers is handling messages' do
-    def build_message queue, explicit_worker = nil
+    def build_message(queue, explicit_worker = nil)
       attributes = {}
-      attributes['shoryuken_class'] = {
-        string_value: explicit_worker.to_s,
-        data_type: 'String' } if explicit_worker
-
-      double Shoryuken::Message,
-        body: 'test',
-        message_attributes: attributes,
-        message_id: SecureRandom.uuid
+
+      if explicit_worker
+        attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' }
+      end
+
+      double(Shoryuken::Message,
+             body: 'test',
+             message_attributes: attributes,
+             message_id: SecureRandom.uuid)
     end
 
     context 'a batch of messages is being processed' do

data/spec/shoryuken/environment_loader_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::EnvironmentLoader do
+  subject { described_class.new({}) }
+
+  describe '#parse_queues' do
+    before do
+      # TODO proper test other methods
+      allow(subject).to receive(:load_rails).with(anything)
+      allow(subject).to receive(:prefix_active_job_queue_names)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+    end
+
+    it 'parses' do
+      Shoryuken.options[:queues] = ['queue_1']
+      subject.load
+
+      expect(Shoryuken.queues).to eq(%w(queue_1))
+    end
+
+    context 'with priority' do
+      it 'parses' do
+        Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
+        subject.load
+
+        expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
+      end
+    end
+  end
+end

data/spec/shoryuken/fetcher_spec.rb

@@ -3,69 +3,35 @@ require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
 describe Shoryuken::Fetcher do
-  let(:manager)    { double Shoryuken::Manager }
-  let(:queue)      { double Shoryuken::Queue }
+  let(:queue)      { instance_double('Shoryuken::Queue') }
   let(:queue_name) { 'default' }
+  let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
 
   let(:sqs_msg) do
-    double Shoryuken::Message,
+    double(Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
-      message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
-  end
-
-  subject { described_class.new(manager) }
-
-  before do
-    allow(manager).to receive(:async).and_return(manager)
-    allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+      message_id: 'fc754df79cc24c4196ca5996a44b771e',
+          )
   end
 
+  subject { described_class.new }
 
   describe '#fetch' do
-    it 'calls pause when no message' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).and_return([])
-
-      expect(manager).to receive(:pause_queue!).with(queue_name)
-      expect(manager).to receive(:dispatch)
-
-      subject.fetch(queue_name, 1)
-    end
-
-    it 'assigns messages' do
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
-
-      expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-      expect(manager).to receive(:assign).with(queue_name, sqs_msg)
-      expect(manager).to receive(:dispatch)
-
-      subject.fetch(queue_name, 5)
+    it 'calls Shoryuken::Client to receive messages' do
+      expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+      expect(queue).to receive(:receive_messages).
+        with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
+        and_return([])
+      subject.fetch(queue_config, 1)
     end
 
-    it 'assigns messages in batch' do
-      TestWorker.get_shoryuken_options['batch'] = true
-
-      allow(queue).to receive(:receive_messages).with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
-
-      expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-      expect(manager).to receive(:assign).with(queue_name, [sqs_msg])
-      expect(manager).to receive(:dispatch)
-
-      subject.fetch(queue_name, 5)
-    end
-
-    context 'when worker not found' do
-      let(:queue_name) { 'notfound' }
-
-      it 'ignores batch' do
-        allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
-
-        expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
-        expect(manager).to receive(:assign).with(queue_name, sqs_msg)
-        expect(manager).to receive(:dispatch)
-
-        subject.fetch(queue_name, 5)
-      end
+    it 'maxes messages to receive to 10 (SQS limit)' do
+      allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+      expect(queue).to receive(:receive_messages).
+        with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
+        and_return([])
+      subject.fetch(queue_config, 20)
     end
   end
 end

data/spec/shoryuken/manager_spec.rb

@@ -1,122 +1,81 @@
 require 'spec_helper'
 require 'shoryuken/manager'
 
+RSpec::Matchers.define :queue_config_of do |expected|
+  match do |actual|
+    actual.name == expected
+  end
+end
+
 RSpec.describe Shoryuken::Manager do
-  subject do
-    condvar = double(:condvar)
-    allow(condvar).to receive(:signal).and_return(nil)
-    Shoryuken::Manager.new(condvar)
+  let(:queue) { 'default' }
+  let(:queues) { [queue] }
+  let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+  let(:fetcher) { Shoryuken::Fetcher.new }
+  let(:concurrency) { 1 }
+
+  subject { Shoryuken::Manager.new(fetcher, polling_strategy) }
+
+  before(:each) do
+    Shoryuken.options[:concurrency] = concurrency
+  end
+
+  after(:each) do
+    Shoryuken.options[:concurrency] = 1
+    TestWorker.get_shoryuken_options['batch'] = false
   end
 
   describe 'Invalid concurrency setting' do
     it 'raises ArgumentError if concurrency is not positive number' do
       Shoryuken.options[:concurrency] = -1
-      expect { Shoryuken::Manager.new(nil) }
+      expect { Shoryuken::Manager.new(nil, nil) }
         .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
     end
-
   end
 
-  describe 'Auto Scaling' do
-    it 'decreases weight' do
-      queue1 = 'shoryuken'
-      queue2 = 'uppercut'
-
-      Shoryuken.queues.clear
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue2
-
-      expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
-
-      subject.pause_queue!(queue1)
-
-      expect(subject.instance_variable_get('@queues')).to eq [queue2]
+  describe '#start' do
+    xit 'pauses when there are no active queues' do
+      expect(polling_strategy).to receive(:next_queue).and_return(nil)
+      expect_any_instance_of(described_class).to receive(:after)
+      subject.start
     end
 
-    it 'increases weight' do
-      queue1 = 'shoryuken'
-      queue2 = 'uppercut'
-
-      Shoryuken.queues.clear
-      # [shoryuken, 3]
-      # [uppercut,  1]
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue2
-
-      expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
-      subject.pause_queue!(queue1)
-      expect(subject.instance_variable_get('@queues')).to eq [queue2]
-
-      subject.rebalance_queue_weight!(queue1)
-      expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
-
-      subject.rebalance_queue_weight!(queue1)
-      expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
-
-      subject.rebalance_queue_weight!(queue1)
-      expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
+    xit 'calls dispatch_batch if worker wants batches' do
+      TestWorker.get_shoryuken_options['batch'] = true
+      expect_any_instance_of(described_class).to receive(:dispatch_batch).with(queue_config_of(queue))
+      expect(subject).to receive(:dispatch_later)
+      subject.start
     end
 
-    it 'adds queue back' do
-      queue1 = 'shoryuken'
-      queue2 = 'uppercut'
-
-      Shoryuken.queues.clear
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue2
-
-      Shoryuken.options[:delay] = 0.1
-
-      fetcher = double('Fetcher').as_null_object
-      subject.fetcher = fetcher
-
-      subject.pause_queue!(queue1)
-      expect(subject.instance_variable_get('@queues')).to eq [queue2]
-
-      sleep 0.5
-
-      expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
+    xit 'calls dispatch_single_messages if worker wants single messages' do
+      expect_any_instance_of(described_class).to receive(:dispatch_single_messages).
+        with(queue_config_of(queue))
+      expect(subject).to receive(:dispatch_later)
+      subject.start
     end
   end
 
-  describe '#next_queue' do
-    it 'returns queues' do
-      queue1 = 'shoryuken'
-      queue2 = 'uppercut'
-
-      Shoryuken.queues.clear
-
-      Shoryuken.register_worker queue1, TestWorker
-      Shoryuken.register_worker queue2, TestWorker
-
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue2
-
-      expect(subject.send :next_queue).to eq queue1
-      expect(subject.send :next_queue).to eq queue2
+  describe '#dispatch_batch' do
+    it 'assings batch as a single message' do
+      q = polling_strategy.next_queue
+      messages = [1, 2, 3]
+      expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+      expect_any_instance_of(described_class).to receive(:assign).with(q.name, messages)
+      subject.send(:dispatch_batch, q)
     end
+  end
 
-    it 'skips when no worker' do
-      queue1 = 'shoryuken'
-      queue2 = 'uppercut'
-
-      Shoryuken.queues.clear
-
-      Shoryuken.register_worker queue2, TestWorker
-
-      Shoryuken.queues << queue1
-      Shoryuken.queues << queue2
-
-      expect(subject.send :next_queue).to eq queue2
-      expect(subject.send :next_queue).to eq queue2
+  describe '#dispatch_single_messages' do
+    let(:concurrency) { 3 }
+
+    it 'assings messages from batch one by one' do
+      q = polling_strategy.next_queue
+      messages = [1, 2, 3]
+      expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
+      expect_any_instance_of(described_class).to receive(:assign).with(q.name, 1)
+      expect_any_instance_of(described_class).to receive(:assign).with(q.name, 2)
+      expect_any_instance_of(described_class).to receive(:assign).with(q.name, 3)
+      subject.send(:dispatch_single_messages, q)
     end
   end
 end