shoryuken 0.0.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93f2193b2c553192ca3d38d1dbcd981bbaa35eeb
-  data.tar.gz: d46dcc643bb7fffeb78f4e2f6ca7f4e1b236d688
+  metadata.gz: 9072ce45c134cd9563ee7dddac27bc7d19733cde
+  data.tar.gz: 2b48cb3b0d594ef1111456d0f9101ce154056995
 SHA512:
-  metadata.gz: 5827e0327eba8f21ab4156d197ea0fd5a8587aa86afda60d14ffc9269e5606aa9688060d59d711821bc117d69ae49a41c0837aaea09433dbb3081fc66b36b9d3
-  data.tar.gz: fcc6b86772574af993cf9145f4d1b0f5bd3e4900937f13c2328ab7dd292dc9bbe1c9e502ea5826220734f091a6ae9d759c0ec1e836c10ce85858925071d94f01
+  metadata.gz: 62bde57b564da861f90c2a68816a69743766f1120dd146817f884b2d4d2e5e5119dcef8858516b7d2581517a04177ec0edb151a97f254c6185ae9092c96efab2
+  data.tar.gz: e011b7d261eaf6e4b87d7dc322d2f458da09fdcd5074eac90eedb8101c082b2987f303b557ba20d90c89b321a7d4df7d613852890b6888263ccbc8adb1bf77db

data/.gitignore

@@ -23,3 +23,4 @@ mkmf.log
 shoryuken.yml
 *.pid
 *.log
+.env

data/.hound.yml

@@ -0,0 +1,2 @@
+StringLiterals:
+  EnforcedStyle: single_quotes

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   # - 1.9.2
   - 2.0.0
   - 2.1.0
+  - 2.2.0
   # - ruby-head
   # - jruby-19mode
   # - jruby-head
@@ -13,4 +14,4 @@ notifications:
     on_success: change
     on_failure: always
 
-script: bundle exec rspec spec
+script: SPEC_ALL=true bundle exec rspec spec

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in shoryuken.gemspec
 gemspec
+
+gem 'codeclimate-test-reporter', group: :test, require: nil

data/README.md

@@ -5,6 +5,7 @@
 Shoryuken _sho-ryu-ken_ is a super-efficient [AWS SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
+[![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
 
 ## Key features
 
@@ -15,8 +16,8 @@ Yeah, Shoryuken load balances the messages consumption!
 Given this configuration:
 
 ```yaml
-concurrency: 25,
-delay: 25,
+concurrency: 25
+delay: 25
 queues:
   - [high_priority, 6]
   - [default, 2]
@@ -37,6 +38,10 @@ If all queues get empty, all processors will be changed to the waiting state and
 
 To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
 
+## Requirements
+
+Ruby 2.0 or greater. Ruby 1.9 is no longer supported.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -109,19 +114,26 @@ aws:
   access_key_id:      ...       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
   secret_access_key:  ...       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
   region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
-  receive_message:              # See http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/Queue.html#receive_message-instance_method
+  receive_message:              # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Queue.html#receive_messages-instance_method
     # wait_time_seconds: N      # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
-    attributes:
-      - receive_count
-      - sent_at
-concurrency: 25,  # The number of allocated threads to process messages. Default 25
-delay: 25,        # The delay in seconds to pause a queue when it's empty. Default 0
+    attribute_names:
+      - ApproximateReceiveCount
+      - SentTimestamp
+concurrency: 25  # The number of allocated threads to process messages. Default 25
+delay: 25        # The delay in seconds to pause a queue when it's empty. Default 0
 queues:
   - [high_priority, 6]
   - [default, 2]
   - [low_priority, 1]
 ```
 
+The ```aws``` section is used to configure both the Aws objects used by Shoryuken internally, and also to set up some Shoryuken-specific config. The Shoryuken-specific keys are listed below, and you can expect any other key defined in that block to be passed on untouched to ```Aws::SQS::Client#initialize```:
+
+- ```account_id``` is used when generating SNS ARNs
+- ```sns_endpoint``` can be used to explicitly override the SNS endpoint
+- ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
+- ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Queue.html#receive_messages-instance_method
+
 ### Rails Integration
 
 You can tell Shoryuken to load your Rails application by passing the `-R` or `--rails` flag to the "shoryuken" command.

data/Rakefile

@@ -11,11 +11,11 @@ task :console do
   if File.exist? config_file
     config = YAML.load File.read(config_file)
 
-    AWS.config(config['aws'])
+    Aws.config = config['aws']
   end
 
   def push(queue, message)
-    Shoryuken::Client.queues(queue).send_message message
+    Shoryuken::Client.queues(queue).send_message(message_body: message)
   end
 
   ARGV.clear

data/examples/bootstrap_queues.rb

@@ -4,17 +4,25 @@ require 'shoryuken'
 # load SQS credentials
 config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
 
-AWS.config(config['aws'])
+Aws.config = config['aws']
 
-sqs = AWS::SQS.new
+sqs = Aws::SQS::Client.new
 
-# create a queue and a respective dead letter queue
-# after 7 attempts SQS will move the message to the dead letter queue
+default_queue_url = sqs.create_queue(queue_name: 'default').queue_url
 
-dl_name = 'default_failures'
-dl = sqs.queues.create(dl_name)
+if sqs.config['endpoint'] =~ /amazonaws.com/
+  # create a dead letter queue
+  # after 7 attempts SQS will move the message to the dead letter queue
 
-options = {}
-options[:redrive_policy] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dl.arn}"}"}
+  dead_letter_queue_url = sqs.create_queue(queue_name: 'default_failures').queue_url
 
-sqs.queues.create('default', options)
+  dead_letter_queue_arn = sqs.get_queue_attributes(
+    queue_url: dead_letter_queue_url,
+    attribute_names: %w(QueueArn)
+  ).attributes['QueueArn']
+
+  attributes = {}
+  attributes['RedrivePolicy'] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"}}
+
+  sqs.set_queue_attributes queue_url: default_queue_url, attributes: attributes
+end

data/lib/shoryuken.rb

@@ -1,18 +1,22 @@
 require 'yaml'
-require 'aws-sdk-v1'
+require 'aws-sdk-core'
+require 'aws-sdk-resources'
 require 'time'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
 require 'shoryuken/util'
+require 'shoryuken/logging'
+require 'shoryuken/environment_loader'
 require 'shoryuken/client'
 require 'shoryuken/worker'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
-require 'shoryuken/logging'
 require 'shoryuken/middleware/chain'
 require 'shoryuken/middleware/server/auto_delete'
 require 'shoryuken/middleware/server/timing'
+require 'shoryuken/sns_arn'
+require 'shoryuken/topic'
 
 module Shoryuken
   DEFAULTS = {
@@ -57,7 +61,7 @@ module Shoryuken
     #   end
     # end
     def configure_server
-      yield self
+      yield self if server?
     end
 
     def server_middleware
@@ -79,6 +83,22 @@ module Shoryuken
       @@default_worker_options = options
     end
 
+    def on_aws_initialization(&block)
+      @aws_initialization_callback = block
+    end
+
+    def on_start(&block)
+      @start_callback = block
+    end
+
+    def on_stop(&block)
+      @stop_callback = block
+    end
+
+    attr_reader :aws_initialization_callback,
+                :start_callback,
+                :stop_callback
+
     private
 
     def default_server_middleware
@@ -91,6 +111,10 @@ module Shoryuken
         end
       end
     end
+
+    def server?
+      defined?(Shoryuken::CLI)
+    end
   end
 end
 

data/lib/shoryuken/cli.rb

@@ -20,19 +20,19 @@ module Shoryuken
       self_read, self_write = IO.pipe
 
       %w[INT TERM USR1 USR2 TTIN].each do |sig|
-        trap sig do
-          self_write.puts(sig)
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
         end
       end
 
-      setup_options(args) do |cli_options|
-        # this needs to happen before configuration is parsed, since it may depend on Rails env
-        initialize_logger(cli_options)
-        load_rails if cli_options[:rails]
-      end
-      require_workers
-      validate!
-      patch_deprecated_workers!
+      options = parse_cli_args(args)
+
+      EnvironmentLoader.load(options)
+
       daemonize
       write_pid
       load_celluloid
@@ -40,6 +40,11 @@ module Shoryuken
       require 'shoryuken/launcher'
       @launcher = Shoryuken::Launcher.new
 
+      if callback = Shoryuken.start_callback
+        logger.info "Calling Shoryuken.on_start block"
+        callback.call
+      end
+
       begin
         launcher.run
 
@@ -67,26 +72,6 @@ module Shoryuken
       require 'shoryuken/manager'
     end
 
-    def load_rails
-      # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
-
-      require 'rails'
-      if ::Rails::VERSION::MAJOR < 4
-        require File.expand_path("config/environment.rb")
-        ::Rails.application.eager_load!
-      else
-        # Painful contortions, see 1791 for discussion
-        require File.expand_path("config/application.rb")
-        ::Rails::Application.initializer "shoryuken.eager_load" do
-          ::Rails.application.config.eager_load = true
-        end
-        require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
-        require File.expand_path("config/environment.rb")
-      end
-
-      logger.info "Rails environment loaded"
-    end
-
     def daemonize
       return unless Shoryuken.options[:daemon]
 
@@ -114,8 +99,6 @@ module Shoryuken
         io.sync = true
       end
       $stdin.reopen('/dev/null')
-
-      initialize_logger
     end
 
     def write_pid
@@ -126,8 +109,8 @@ module Shoryuken
       end
     end
 
-    def parse_options(argv)
-      opts = {}
+    def parse_cli_args(argv)
+      opts = { queues: [] }
 
       @parser = OptionParser.new do |o|
         o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
@@ -140,7 +123,7 @@ module Shoryuken
 
         o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
           queue, weight = arg.split(',')
-          parse_queue queue, weight
+          opts[:queues] << [queue, weight]
         end
 
         o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
@@ -213,96 +196,5 @@ module Shoryuken
         raise Interrupt
       end
     end
-
-    def setup_options(args)
-      options = parse_options(args)
-
-      # yield parsed options in case we need to do more setup before configuration is parsed
-      yield(options) if block_given?
-
-      config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
-
-      Shoryuken.options.merge!(config)
-
-      Shoryuken.options.merge!(options)
-
-      parse_queues
-    end
-
-    def parse_config(config_file)
-      if File.exist?(config_file)
-        YAML.load(ERB.new(IO.read(config_file)).result)
-      else
-        raise ArgumentError, "Config file #{config_file} does not exist"
-      end
-    end
-
-    def initialize_logger(options = Shoryuken.options)
-      Shoryuken::Logging.initialize_logger(options[:logfile]) if options[:logfile]
-
-      Shoryuken.logger.level = Logger::DEBUG if options[:verbose]
-    end
-
-    def validate!
-      raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
-
-      all_queues = Shoryuken.queues
-      queues_with_workers = Shoryuken.worker_registry.queues
-
-      unless defined?(::ActiveJob)
-        (all_queues - queues_with_workers).each do |queue|
-          logger.warn "No worker supplied for '#{queue}'"
-        end
-      end
-
-      initialize_aws
-
-      Shoryuken.queues.uniq.each do |queue|
-        # validate all queues and AWS credentials consequently
-        begin
-          Shoryuken::Client.queues queue
-        rescue AWS::SQS::Errors::NonExistentQueue => e
-          raise ArgumentError, "Queue '#{queue}' does not exist"
-        rescue => e
-          raise
-        end
-      end
-    end
-
-    def initialize_aws
-      # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-      # when not explicit supplied
-      AWS.config Shoryuken.options[:aws] if Shoryuken.options[:aws]
-    end
-
-    def require_workers
-      require Shoryuken.options[:require] if Shoryuken.options[:require]
-    end
-
-    def parse_queues
-      Shoryuken.options[:queues].to_a.each { |queue_and_weight| parse_queue *queue_and_weight }
-    end
-
-    def parse_queue(queue, weight = nil)
-      [weight.to_i, 1].max.times { Shoryuken.queues << queue }
-    end
-
-    def patch_deprecated_workers!
-      Shoryuken.worker_registry.queues.each do |queue|
-        Shoryuken.worker_registry.workers(queue).each do |worker_class|
-          if worker_class.instance_method(:perform).arity == 1
-            logger.warn "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)"
-
-            worker_class.class_eval do
-              alias_method :deprecated_perform, :perform
-
-              def perform(sqs_msg, body = nil)
-                deprecated_perform(sqs_msg)
-              end
-            end
-          end
-        end
-      end
-    end
   end
 end

data/lib/shoryuken/client.rb

@@ -1,29 +1,43 @@
 module Shoryuken
   class Client
     @@queues = {}
-    @@visibility_timeouts = {}
+    @@topics = {}
 
     class << self
-      def queues(queue)
-        @@queues[queue.to_s] ||= sqs.queues.named(queue)
+      def queues(name)
+        @@queues[name.to_s] ||= sqs_resource.get_queue_by_name(queue_name: name)
       end
 
-      def visibility_timeout(queue)
-        @@visibility_timeouts[queue.to_s] ||= queues(queue).visibility_timeout
+      def sns
+        @sns ||= Aws::SNS::Client.new(aws_client_options(:sns_endpoint))
       end
 
-      def receive_message(queue, options = {})
-        queues(queue).receive_message(Hash(options))
+      def sns_arn
+        @sns_arn ||= SnsArn
       end
 
-      def send_message(queue, body, options = {})
-        body = JSON.dump(body) if body.is_a?(Hash)
+      def sqs
+        @sqs ||= Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
+      end
 
-        queues(queue).send_message(body, options)
+      def sqs_resource
+        @sqs_resource ||= Aws::SQS::Resource.new(client: sqs)
       end
 
-      def sqs
-        @sqs ||= AWS::SQS.new
+      def topics(name)
+        @@topics[name.to_s] ||= Topic.new(name, sns)
+      end
+
+      attr_accessor :account_id
+      attr_writer :sns, :sqs, :sqs_resource, :sns_arn
+
+      private
+
+      def aws_client_options service_endpoint_key
+        explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key]
+        options = {}
+        options[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
+        options
       end
     end
   end