shoryuken 2.1.0 → 3.0.0

data/test_workers/endless_uninterruptive_worker.rb

@@ -0,0 +1,44 @@
+class EndlessUninterruptiveWorker
+  include Shoryuken::Worker
+
+  # Usage:
+  # QUEUE="super-q"
+  # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
+  # bundle exec ./bin/shoryuken -r ./examples/endless_interruptive_worker.rb -q $QUEUE -c 8
+
+  class << self
+    def queue
+      ENV['QUEUE'] || 'default'
+    end
+
+    def max_execution_time
+      ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
+    end
+
+    def rng
+      @rng ||= Random.new
+    end
+
+    # returns a random number between 0 and 100
+    def random_number(hi = 1000)
+      (rng.rand * hi).to_i
+    end
+  end
+
+  def perform(sqs_msg, body)
+    Shoryuken.logger.info("Received message: '#{body}'")
+
+    execution_ms = self.class.random_number(self.class.max_execution_time)
+    Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")
+    end_time = Time.now + execution_ms.to_f / 1000
+    while Time.now < end_time do
+      # burn metal
+    end
+
+    new_body = "#{execution_ms}-" + body.to_s
+
+    self.class.perform_async(new_body.slice(0, 512))
+  end
+
+  shoryuken_options queue: queue, auto_delete: true
+end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Pablo Cantero
-- Mario Kostelac
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-03 00:00:00.000000000 Z
+date: 2017-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -68,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -82,51 +81,50 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: celluloid
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
 description: Shoryuken is a super efficient AWS SQS thread based message processor
 email:
 - pablo@pablocantero.com
-- mariokostelac@gmail.com
 executables:
 - shoryuken
 extensions: []
@@ -142,12 +140,12 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/cli/base.rb
+- bin/cli/sqs.rb
 - bin/shoryuken
 - examples/bootstrap_queues.rb
 - examples/default_worker.rb
 - lib/shoryuken.rb
-- lib/shoryuken/aws_config.rb
-- lib/shoryuken/cli.rb
 - lib/shoryuken/client.rb
 - lib/shoryuken/core_ext.rb
 - lib/shoryuken/default_worker_registry.rb
@@ -164,10 +162,10 @@ files:
 - lib/shoryuken/middleware/server/auto_extend_visibility.rb
 - lib/shoryuken/middleware/server/exponential_backoff_retry.rb
 - lib/shoryuken/middleware/server/timing.rb
+- lib/shoryuken/polling.rb
 - lib/shoryuken/processor.rb
 - lib/shoryuken/queue.rb
-- lib/shoryuken/sns_arn.rb
-- lib/shoryuken/topic.rb
+- lib/shoryuken/runner.rb
 - lib/shoryuken/util.rb
 - lib/shoryuken/version.rb
 - lib/shoryuken/worker.rb
@@ -176,10 +174,10 @@ files:
 - shoryuken.jpg
 - spec/integration/launcher_spec.rb
 - spec/shoryuken.yml
-- spec/shoryuken/cli_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
+- spec/shoryuken/environment_loader_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
@@ -187,15 +185,16 @@ files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
+- spec/shoryuken/polling_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
-- spec/shoryuken/sns_arn_spec.rb
-- spec/shoryuken/topic_spec.rb
+- spec/shoryuken/runner_spec.rb
 - spec/shoryuken/util_spec.rb
 - spec/shoryuken/worker_spec.rb
-- spec/shoryuken_endpoint.yml
 - spec/shoryuken_spec.rb
 - spec/spec_helper.rb
+- test_workers/endless_interruptive_worker.rb
+- test_workers/endless_uninterruptive_worker.rb
 homepage: https://github.com/phstc/shoryuken
 licenses:
 - LGPL-3.0
@@ -216,17 +215,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Shoryuken is a super efficient AWS SQS thread based message processor
 test_files:
 - spec/integration/launcher_spec.rb
 - spec/shoryuken.yml
-- spec/shoryuken/cli_spec.rb
 - spec/shoryuken/client_spec.rb
 - spec/shoryuken/core_ext_spec.rb
 - spec/shoryuken/default_worker_registry_spec.rb
+- spec/shoryuken/environment_loader_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
@@ -234,12 +233,11 @@ test_files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
+- spec/shoryuken/polling_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
-- spec/shoryuken/sns_arn_spec.rb
-- spec/shoryuken/topic_spec.rb
+- spec/shoryuken/runner_spec.rb
 - spec/shoryuken/util_spec.rb
 - spec/shoryuken/worker_spec.rb
-- spec/shoryuken_endpoint.yml
 - spec/shoryuken_spec.rb
 - spec/spec_helper.rb

data/lib/shoryuken/aws_config.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-module Shoryuken
-  class AwsConfig
-    class << self
-      attr_writer :options
-
-      def options
-        @options ||= {}
-      end
-
-      def setup(hash)
-        # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-        # when not explicit supplied
-        return if hash.empty?
-
-        self.options = hash
-
-        shoryuken_keys = %w(
-          account_id
-          sns_endpoint
-          sqs_endpoint
-          receive_message
-        ).map(&:to_sym)
-
-        @aws_options = hash.reject do |k, _|
-          shoryuken_keys.include?(k)
-        end
-
-        # assume credentials based authentication
-        credentials = Aws::Credentials.new(
-          @aws_options.delete(:access_key_id),
-          @aws_options.delete(:secret_access_key)
-        )
-
-        # but only if the configuration options have valid values
-        @aws_options.merge!(credentials: credentials) if credentials.set?
-
-        if (callback = Shoryuken.aws_initialization_callback)
-          Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
-          callback.call(@aws_options)
-        end
-      end
-
-      def sns
-        Aws::SNS::Client.new(aws_client_options(:sns_endpoint))
-      end
-
-      def sqs
-        Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
-      end
-
-      private
-
-      def aws_client_options(service_endpoint_key)
-        environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
-        explicit_endpoint = options[service_endpoint_key] || environment_endpoint
-        endpoint = {}.tap do |hash|
-          hash[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
-        end
-        @aws_options.to_h.merge(endpoint)
-      end
-    end
-  end
-end

data/lib/shoryuken/cli.rb

@@ -1,215 +0,0 @@
-$stdout.sync = true
-
-require 'singleton'
-require 'optparse'
-require 'erb'
-
-require 'shoryuken'
-
-module Shoryuken
-  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
-  class Shutdown < Interrupt; end
-
-  class CLI
-    include Util
-    include Singleton
-
-    attr_accessor :launcher
-
-    def run(args)
-      self_read, self_write = IO.pipe
-
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
-        begin
-          trap sig do
-            self_write.puts(sig)
-          end
-        rescue ArgumentError
-          puts "Signal #{sig} not supported"
-        end
-      end
-
-      options = parse_cli_args(args)
-
-      loader = EnvironmentLoader.setup_options(options)
-
-      # When cli args exist, override options in config file
-      Shoryuken.options.merge!(options)
-
-      daemonize(Shoryuken.options)
-      write_pid(Shoryuken.options)
-
-      loader.load
-
-      load_celluloid
-
-      require 'shoryuken/launcher'
-      @launcher = Shoryuken::Launcher.new
-
-      if callback = Shoryuken.start_callback
-        logger.info { 'Calling Shoryuken.on_start block' }
-        callback.call
-      end
-
-      fire_event(:startup)
-
-      begin
-        launcher.run
-
-        while (readable_io = IO.select([self_read]))
-          signal = readable_io.first[0].gets.strip
-          handle_signal(signal)
-        end
-      rescue Interrupt
-        launcher.stop(shutdown: true)
-        exit 0
-      end
-    end
-
-    private
-
-    def load_celluloid
-      require 'celluloid/current'
-      Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
-
-      require 'shoryuken/manager'
-    end
-
-    def celluloid_loaded?
-      defined?(::Celluloid)
-    end
-
-    def daemonize(options)
-      return unless options[:daemon]
-
-      fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
-
-      if celluloid_loaded?
-        # Celluloid can't be loaded until after we've daemonized
-        # because it spins up threads and creates locks which get
-        # into a very bad state if forked.
-        raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization"
-      end
-
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
-
-      Process.daemon(true, true)
-
-      files_to_reopen.each do |file|
-        begin
-          file.reopen file.path, 'a+'
-          file.sync = true
-        rescue ::Exception
-        end
-      end
-
-      [$stdout, $stderr].each do |io|
-        File.open(options[:logfile], 'ab') do |f|
-          io.reopen(f)
-        end
-        io.sync = true
-      end
-      $stdin.reopen('/dev/null')
-    end
-
-    def write_pid(options)
-      if (path = options[:pidfile])
-        File.open(path, 'w') do |f|
-          f.puts Process.pid
-        end
-      end
-    end
-
-    def parse_cli_args(argv)
-      opts = {}
-
-      @parser = OptionParser.new do |o|
-        o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
-          opts[:concurrency] = Integer(arg)
-        end
-
-        o.on '-d', '--daemon', 'Daemonize process' do |arg|
-          opts[:daemon] = arg
-        end
-
-        o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
-          queue, weight = arg.split(',')
-          opts[:queues] = [] unless opts[:queues]
-          opts[:queues] << [queue, weight]
-        end
-
-        o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
-          opts[:require] = arg
-        end
-
-        o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
-          opts[:config_file] = arg
-        end
-
-        o.on '-R', '--rails', 'Load Rails' do |arg|
-          opts[:rails] = arg
-        end
-
-        o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
-          opts[:logfile] = arg
-        end
-
-        o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
-          opts[:pidfile] = arg
-        end
-
-        o.on '-v', '--verbose', 'Print more verbose output' do |arg|
-          opts[:verbose] = arg
-        end
-
-        o.on '-V', '--version', 'Print version and exit' do
-          puts "Shoryuken #{Shoryuken::VERSION}"
-          exit 0
-        end
-      end
-
-      @parser.banner = 'shoryuken [options]'
-      @parser.on_tail '-h', '--help', 'Show help' do
-        logger.info { @parser }
-        exit 1
-      end
-      @parser.parse!(argv)
-      opts
-    end
-
-    def handle_signal(sig)
-      logger.info { "Got #{sig} signal" }
-
-      case sig
-      when 'USR1'
-        logger.info { 'Received USR1, will soft shutdown down' }
-
-        launcher.stop
-        fire_event(:quiet, true)
-        exit 0
-      when 'TTIN'
-        Thread.list.each do |thread|
-          logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
-          if thread.backtrace
-            logger.info { thread.backtrace.join("\n") }
-          else
-            logger.info { '<no backtrace available>' }
-          end
-        end
-
-        ready  = launcher.manager.instance_variable_get(:@ready).size
-        busy   = launcher.manager.instance_variable_get(:@busy).size
-        queues = launcher.manager.instance_variable_get(:@queues)
-
-        logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}" }
-      else
-        logger.info { "Received #{sig}, will shutdown down" }
-
-        raise Interrupt
-      end
-    end
-  end
-end

data/lib/shoryuken/sns_arn.rb

@@ -1,27 +0,0 @@
-module Shoryuken
-  class SnsArn
-    def initialize(topic)
-      @topic = topic
-    end
-
-    def to_s
-      @arn ||= "arn:aws:sns:#{region}:#{account_id}:#{@topic}"
-    end
-
-    private
-
-    def account_id
-      Shoryuken::Client.account_id.tap do |account_id|
-        if account_id.to_s.empty?
-          fail 'To generate SNS ARNs, you must assign an :account_id in your Shoryuken::Client.'
-        end
-      end
-    end
-
-    def region
-      Aws.config.fetch(:region) do
-        fail 'To generate SNS ARNs, you must include a :region in your AWS config.'
-      end
-    end
-  end
-end

data/lib/shoryuken/topic.rb

@@ -1,17 +0,0 @@
-module Shoryuken
-  class Topic
-    def initialize(name, sns)
-      @name, @sns = name, sns
-    end
-
-    def arn
-      @arn ||= Client.sns_arn.new(@name).to_s
-    end
-
-    def send_message(body, options = {})
-      body = JSON.dump(body) if body.is_a?(Hash)
-
-      @sns.publish(topic_arn: arn, message: body)
-    end
-  end
-end

data/spec/shoryuken/sns_arn_spec.rb

@@ -1,42 +0,0 @@
-require 'spec_helper'
-
-describe Shoryuken::SnsArn do
-  let(:account_id) { '1234567890' }
-  let(:region) { 'eu-west-1' }
-  let(:topic) { 'topic-x' }
-
-  before do
-    Shoryuken::Client.account_id = account_id
-    Aws.config = { region: region }
-  end
-
-  subject { described_class.new(topic).to_s }
-
-  describe '#to_s' do
-    context 'when the Aws config includes all the information necessary' do
-      it 'generates an SNS arn' do
-        expect(subject).to eq('arn:aws:sns:eu-west-1:1234567890:topic-x')
-      end
-    end
-
-    context 'when the Aws config does not include the account id' do
-      before do
-        Shoryuken::Client.account_id = nil
-      end
-
-      it 'fails' do
-        expect { subject }.to raise_error(/an :account_id/)
-      end
-    end
-
-    context 'when the Aws config does not include the region' do
-      before do
-        Aws.config.delete :region
-      end
-
-      it 'fails' do
-        expect { subject }.to raise_error(/a :region/)
-      end
-    end
-  end
-end

data/spec/shoryuken/topic_spec.rb

@@ -1,32 +0,0 @@
-require 'spec_helper'
-
-describe Shoryuken::Topic do
-  let(:sns) { Aws::SNS::Client.new stub_responses: true }
-  let(:topic_arn) { 'arn:aws:sns:us-east-1:0987654321:shoryuken' }
-  let(:topic_name) { 'shoryuken' }
-
-  before do
-    Shoryuken::Client.account_id = '0987654321'
-    Aws.config = { region: 'us-east-1' }
-  end
-
-  subject { described_class.new(topic_name, sns) }
-
-  describe '#send_message' do
-    it 'enqueues a message' do
-      sns.stub_responses(:publish, { message_id: 'msg1' })
-      expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: 'test')
-
-      subject.send_message('test')
-    end
-
-    it 'parses as JSON by default' do
-      msg = { field: 'test', other_field: 'other' }
-
-      sns.stub_responses(:publish, { message_id: 'msg2' })
-      expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: JSON.dump(msg))
-
-      subject.send_message(msg)
-    end
-  end
-end

data/spec/shoryuken_endpoint.yml

@@ -1,6 +0,0 @@
-aws:
-  access_key_id:      <%= ENV['AWS_ACCESS_KEY_ID'] %>
-  secret_access_key:  <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
-  region:             us-east-1
-  sqs_endpoint: https://github.com/phstc/shoryuken:4568
-  sns_endpoint: http://127.0.0.1:4568