shoryuken 2.1.3 → 3.0.0

data/lib/shoryuken/polling.rb

@@ -65,7 +65,6 @@ module Shoryuken
     end
 
     class WeightedRoundRobin < BaseStrategy
-      
       def initialize(queues)
         @initial_queues = queues
         @queues = queues.dup.uniq
@@ -106,7 +105,7 @@ module Shoryuken
         @paused_queues << [Time.now + delay, queue]
         logger.debug "Paused '#{queue}'"
       end
- 
+
       def unpause_queues
         return if @paused_queues.empty?
         return if Time.now < @paused_queues.first[0]
@@ -129,7 +128,6 @@ module Shoryuken
     end
 
     class StrictPriority < BaseStrategy
-
       def initialize(queues)
         # Priority ordering of the queues, highest priority first
         @queues = queues
@@ -178,7 +176,7 @@ module Shoryuken
           return queue unless queue_paused?(queue)
         end
 
-        return nil
+        nil
       end
 
       def queues_unpaused_since?

data/lib/shoryuken/processor.rb

@@ -1,16 +1,11 @@
-require 'json'
-
 module Shoryuken
   class Processor
-    include Celluloid
     include Util
 
     def initialize(manager)
       @manager = manager
     end
 
-    attr_accessor :proxy_id
-
     def process(queue, sqs_msg)
       worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
       body = get_body(worker.class, sqs_msg)
@@ -18,12 +13,8 @@ module Shoryuken
       worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
         worker.perform(sqs_msg, body)
       end
-
-      @manager.async.processor_done(queue, current_actor)
-    end
-
-    def running_thread
-      Thread.current
+    ensure
+      @manager.processor_done(queue)
     end
 
     private

data/lib/shoryuken/queue.rb

@@ -35,9 +35,7 @@ module Shoryuken
     end
 
     def receive_messages(options)
-      client.receive_message(options.merge(queue_url: url)).
-        messages.
-        map { |m| Message.new(client, self, m) }
+      client.receive_message(options.merge(queue_url: url)).messages.map { |m| Message.new(client, self, m) }
     end
 
     def fifo?

data/lib/shoryuken/runner.rb

@@ -0,0 +1,143 @@
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'erb'
+
+require 'shoryuken'
+
+module Shoryuken
+  # rubocop:disable Lint/InheritException
+  # rubocop:disable Metrics/AbcSize
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
+  class Runner
+    include Util
+    include Singleton
+
+    def run(options)
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      loader = EnvironmentLoader.setup_options(options)
+
+      # When cli args exist, override options in config file
+      Shoryuken.options.merge!(options)
+
+      daemonize(Shoryuken.options)
+      write_pid(Shoryuken.options)
+
+      loader.load
+
+      initialize_concurrent_logger
+
+      @launcher = Shoryuken::Launcher.new
+
+      if (callback = Shoryuken.start_callback)
+        logger.info { 'Calling Shoryuken.on_start block' }
+        callback.call
+      end
+
+      fire_event(:startup)
+
+      begin
+        @launcher.run
+
+        while (readable_io = IO.select([self_read]))
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        @launcher.stop(shutdown: true)
+        exit 0
+      end
+    end
+
+    private
+
+    def initialize_concurrent_logger
+      return unless Shoryuken.logger
+
+      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+        Shoryuken.logger.log(level, msg, progname, &block)
+      end
+    end
+
+    def daemonize(options)
+      return unless options[:daemon]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      [$stdout, $stderr].each do |io|
+        File.open(options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+    end
+
+    def write_pid(options)
+      return unless (path = options[:pidfile])
+
+      File.open(path, 'w') { |f| f.puts(Process.pid) }
+    end
+
+    def execute_soft_shutdown
+      logger.info { 'Received USR1, will soft shutdown down' }
+
+      @launcher.stop
+      fire_event(:quiet, true)
+      exit 0
+    end
+
+    def print_threads_backtrace
+      Thread.list.each do |thread|
+        logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
+        if thread.backtrace
+          logger.info { thread.backtrace.join("\n") }
+        else
+          logger.info { '<no backtrace available>' }
+        end
+      end
+    end
+
+    def handle_signal(sig)
+      logger.info { "Got #{sig} signal" }
+
+      case sig
+      when 'USR1' then execute_soft_shutdown
+      when 'TTIN' then print_threads_backtrace
+      when 'USR2'
+        logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
+      else
+        logger.info { "Received #{sig}, will shutdown down" }
+
+        raise Interrupt
+      end
+    end
+  end
+end

data/lib/shoryuken/util.rb

@@ -1,13 +1,5 @@
 module Shoryuken
   module Util
-    def watchdog(last_words)
-      yield
-    rescue => ex
-      logger.error { last_words }
-      logger.error { ex }
-      logger.error { ex.backtrace.join("\n") }
-    end
-
     def logger
       Shoryuken.logger
     end

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '2.1.3'
+  VERSION = '3.0.0'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -63,7 +63,7 @@ module Shoryuken
 
       def normalize_worker_queue!
         queue = @shoryuken_options['queue']
-        if queue.respond_to? :call
+        if queue.respond_to?(:call)
           queue = queue.call
           @shoryuken_options['queue'] = queue
         end

data/shoryuken.gemspec

@@ -6,14 +6,14 @@ require 'shoryuken/version'
 Gem::Specification.new do |spec|
   spec.name          = 'shoryuken'
   spec.version       = Shoryuken::VERSION
-  spec.authors       = ['Pablo Cantero', 'Mario Kostelac']
-  spec.email         = ['pablo@pablocantero.com', 'mariokostelac@gmail.com']
-  spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
+  spec.authors       = ['Pablo Cantero']
+  spec.email         = ['pablo@pablocantero.com']
+  spec.description = spec.summary = 'Shoryuken is a super efficient AWS SQS thread based message processor'
   spec.homepage      = 'https://github.com/phstc/shoryuken'
   spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = %w[shoryuken]
+  spec.executables   = %w(shoryuken)
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'dotenv'
 
   spec.add_dependency 'aws-sdk-core', '~> 2'
-  spec.add_dependency 'celluloid', '~> 0.17'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'thor'
 end

data/spec/integration/launcher_spec.rb

@@ -1,12 +1,11 @@
 require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
+require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
   describe 'Consuming messages', slow: :true do
     before do
-      Shoryuken.options[:aws][:receive_message] = { wait_time_seconds: 5 }
-
       StandardWorker.received_messages = 0
 
       queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
@@ -21,7 +20,9 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     after do
-      queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: StandardWorker.get_shoryuken_options['queue']).queue_url
+      queue_url = Shoryuken::Client.sqs.get_queue_url(
+        queue_name: StandardWorker.get_shoryuken_options['queue']
+      ).queue_url
 
       Shoryuken::Client.sqs.delete_queue queue_url: queue_url
     end

data/spec/shoryuken/client_spec.rb

@@ -1,17 +1,16 @@
 require 'spec_helper'
 
-describe Shoryuken::Client do
+RSpec.describe Shoryuken::Client do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs)         { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name)  { 'shoryuken' }
   let(:queue_url)   { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
-  let(:sqs_endpoint) { 'http://localhost:4568' }
-  let(:sns_endpoint) { 'http://0.0.0.0:4568'   }
 
   describe '.queue' do
     before do
       described_class.sqs = sqs
     end
+
     it 'memoizes queues' do
       sqs.stub_responses(:get_queue_url, { queue_url: queue_url }, { queue_url: 'xyz' })
 
@@ -19,46 +18,4 @@ describe Shoryuken::Client do
       expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
     end
   end
-
-  describe 'environment variable endpoints' do
-    before do
-      ENV['AWS_SQS_ENDPOINT'] = sqs_endpoint
-      ENV['AWS_SNS_ENDPOINT'] = sns_endpoint
-      ENV['AWS_REGION'] = 'us-east-1'
-      Shoryuken.options[:aws] = {}
-      Shoryuken::AwsConfig.options = {}
-    end
-
-    it 'will use config file settings if set' do
-      load_config_file_by_file_name('shoryuken_endpoint.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://github.com/phstc/shoryuken:4568')
-      expect(described_class.sns.config.endpoint.to_s).to eq('http://127.0.0.1:4568')
-    end
-
-    it 'should fallback to environment variable if config file not found or set' do
-      load_config_file_by_file_name(nil)
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'should fallback to environment variable if config file found but settings not set' do
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
-      expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
-    end
-
-    it 'will fallback to default settings if no config file settings or environment variables found' do
-      ENV['AWS_SQS_ENDPOINT'] = nil
-      ENV['AWS_SNS_ENDPOINT'] = nil
-      load_config_file_by_file_name('shoryuken.yml')
-      expect(described_class.sqs.config.endpoint.to_s).to eql('https://sqs.us-east-1.amazonaws.com')
-      expect(described_class.sns.config.endpoint.to_s).to eq('https://sns.us-east-1.amazonaws.com')
-    end
-  end
-
-  def load_config_file_by_file_name(file_name)
-    path_name = file_name ? File.join(File.expand_path('../../..', __FILE__), 'spec', file_name) : nil
-    loader = Shoryuken::EnvironmentLoader.setup_options(config_file: path_name)
-    loader.load
-  end
 end

data/spec/shoryuken/default_worker_registry_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 
-describe Shoryuken::DefaultWorkerRegistry do
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::DefaultWorkerRegistry do
   class RegistryTestWorker
     include Shoryuken::Worker
 
@@ -42,16 +43,17 @@ describe Shoryuken::DefaultWorkerRegistry do
   end
 
   describe 'a registry with workers is handling messages' do
-    def build_message queue, explicit_worker = nil
+    def build_message(queue, explicit_worker = nil)
       attributes = {}
-      attributes['shoryuken_class'] = {
-        string_value: explicit_worker.to_s,
-        data_type: 'String' } if explicit_worker
-
-      double Shoryuken::Message,
-        body: 'test',
-        message_attributes: attributes,
-        message_id: SecureRandom.uuid
+
+      if explicit_worker
+        attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' }
+      end
+
+      double(Shoryuken::Message,
+             body: 'test',
+             message_attributes: attributes,
+             message_id: SecureRandom.uuid)
     end
 
     context 'a batch of messages is being processed' do

data/spec/shoryuken/environment_loader_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::EnvironmentLoader do
+  subject { described_class.new({}) }
+
+  describe '#parse_queues' do
+    before do
+      # TODO proper test other methods
+      allow(subject).to receive(:load_rails).with(anything)
+      allow(subject).to receive(:prefix_active_job_queue_names)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+    end
+
+    it 'parses' do
+      Shoryuken.options[:queues] = ['queue_1']
+      subject.load
+
+      expect(Shoryuken.queues).to eq(%w(queue_1))
+    end
+
+    context 'with priority' do
+      it 'parses' do
+        Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
+        subject.load
+
+        expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
+      end
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -12,21 +12,12 @@ RSpec.describe Shoryuken::Manager do
   let(:queues) { [queue] }
   let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
   let(:fetcher) { Shoryuken::Fetcher.new }
-  let(:condvar) do
-    condvar = double(:condvar)
-    allow(condvar).to receive(:signal).and_return(nil)
-    condvar
-  end
-  let(:async_manager) { instance_double(described_class.name) }
   let(:concurrency) { 1 }
 
-  subject { Shoryuken::Manager.new(condvar) }
+  subject { Shoryuken::Manager.new(fetcher, polling_strategy) }
 
   before(:each) do
     Shoryuken.options[:concurrency] = concurrency
-    subject.fetcher = fetcher
-    subject.polling_strategy = polling_strategy
-    allow_any_instance_of(described_class).to receive(:async).and_return(async_manager)
   end
 
   after(:each) do
@@ -37,31 +28,30 @@ RSpec.describe Shoryuken::Manager do
   describe 'Invalid concurrency setting' do
     it 'raises ArgumentError if concurrency is not positive number' do
       Shoryuken.options[:concurrency] = -1
-      expect { Shoryuken::Manager.new(nil) }
+      expect { Shoryuken::Manager.new(nil, nil) }
         .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
     end
   end
 
-  describe '#dispatch' do
-    it 'pauses when there are no active queues' do
+  describe '#start' do
+    xit 'pauses when there are no active queues' do
       expect(polling_strategy).to receive(:next_queue).and_return(nil)
       expect_any_instance_of(described_class).to receive(:after)
-      subject.dispatch
+      subject.start
     end
 
-    it 'calls dispatch_batch if worker wants batches' do
+    xit 'calls dispatch_batch if worker wants batches' do
       TestWorker.get_shoryuken_options['batch'] = true
       expect_any_instance_of(described_class).to receive(:dispatch_batch).with(queue_config_of(queue))
-      expect_any_instance_of(described_class).to receive(:async).and_return(async_manager)
-      expect(async_manager).to receive(:dispatch)
-      subject.dispatch
+      expect(subject).to receive(:dispatch_later)
+      subject.start
     end
 
-    it 'calls dispatch_single_messages if worker wants single messages' do
+    xit 'calls dispatch_single_messages if worker wants single messages' do
       expect_any_instance_of(described_class).to receive(:dispatch_single_messages).
         with(queue_config_of(queue))
-      expect(async_manager).to receive(:dispatch)
-      subject.dispatch
+      expect(subject).to receive(:dispatch_later)
+      subject.start
     end
   end