shoryuken 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 767009617d3e6954d4b1bec86f33811fdfd6c4e0
-  data.tar.gz: 578e87e222d8f980a72981f75ae57a7ad6a5275f
+  metadata.gz: fd50ddb123d1c14d3ff0a0ac562c3e156afd2dfd
+  data.tar.gz: 9bcd1a88456f1d8454609d7440d209bc78b5a89b
 SHA512:
-  metadata.gz: 6c8493dc99b8b11ee02e7c5ae1d51acf499e60b0ba1b94c2202b40d7e36f4ea3d1ea03ab9f5d2570e7943d87f4cd2b0c54940c62b12c08903f83584967897203
-  data.tar.gz: b04c12cd3ef6efe7ac4ca4a8ab3f6011bee687aa5053db2ed0fd03205d32e76a3cd45809a4aa8e75e8b7f50d311334dd43aa698cc25e366019ddc9d932bd8a94
+  metadata.gz: 64680cf91fbf51372720f872f0ac7cced4d42e2b86c61b66137fd5993f1e58ca2ab86c907ff49eb8c464817c5c4efe53ea75e234a4ee01a4d558164535086c46
+  data.tar.gz: 75fbaab18cd22e600ccd6e705eef80532faf0236a434803cf0678dc14cf6b7fac30e40d0350e8be2b47eb8dd54289667eeccb12a49af0febbcafc0d5594e05f5

data/.codeclimate.yml

@@ -1,8 +1,6 @@
 ---
 engines:
-  brakeman:
-    enabled: true
-  bundler-audit:
+  reek:
     enabled: true
   duplication:
     enabled: true
@@ -11,13 +9,10 @@ engines:
       - ruby
   fixme:
     enabled: true
-  reek:
-    enabled: true
   rubocop:
     enabled: true
 ratings:
   paths:
   - "**.rb"
-# exclude_paths:
-# - "**/vendor/**/*"
-# - "*/spec/**/*"
+exclude_paths:
+- spec/

data/.travis.yml

@@ -14,3 +14,7 @@ before_install:
  - gem update bundler
 after_success:
   - bundle exec codeclimate-test-reporter
+
+addons:
+  code_climate:
+    repo_token: 7709fd21981bb9d2658647a66d959415a1029a83f1c199573828797944f26c52

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [v2.1.2] - 2016-12-22
+- Fix loading `logfile` from shoryuken.yml
+ - [#296](https://github.com/phstc/shoryuken/pull/296)
+
+- Add support for Strict priority polling (pending documentation)
+ - [#288](https://github.com/phstc/shoryuken/pull/288)
+
+- Add `test_workers` for end-to-end testing supporting
+ - [#286](https://github.com/phstc/shoryuken/pull/286)
+
+- Update README documenting `configure_client` and `configure_server`
+ - [#283](https://github.com/phstc/shoryuken/pull/283)
+
+- Fix memory leak caused by async tracking busy threads
+ - [#289](https://github.com/phstc/shoryuken/pull/289)
+
+- Refactor fetcher, polling strategy and manager
+ - [#284](https://github.com/phstc/shoryuken/pull/284)
+
 ## [v2.1.1] - 2016-12-05
 - Fix aws deprecation warning message
  - [#279](https://github.com/phstc/shoryuken/pull/279)

data/README.md

@@ -110,15 +110,6 @@ end
 Sample configuration file `shoryuken.yml`.
 
 ```yaml
-aws:
-  access_key_id:      ...       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
-  secret_access_key:  ...       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
-  region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
-  receive_message:              # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
-    # wait_time_seconds: N      # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
-    attribute_names:
-      - ApproximateReceiveCount
-      - SentTimestamp
 concurrency: 25  # The number of allocated threads to process messages. Default 25
 delay: 25        # The delay in seconds to pause a queue when it's empty. Default 0
 queues:
@@ -127,24 +118,80 @@ queues:
   - [low_priority, 1]
 ```
 
-The ```aws``` section is used to configure both the Aws objects used by Shoryuken internally, and also to set up some Shoryuken-specific config. The Shoryuken-specific keys are listed below, and you can expect any other key defined in that block to be passed on untouched to ```Aws::SQS::Client#initialize```:
+And setup ```aws``` options to use ```configure_client``` in `config/initializers/shoryuken.rb`:
 
-- ```account_id``` is used when generating SNS ARNs
-- ```sns_endpoint``` can be used to explicitly override the SNS endpoint
-- ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
-- ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
+```ruby
+Shoryuken.configure_client do |config|
+  config.aws = {
+    secret_access_key: ...,         # or ENV["AWS_SECRET_ACCESS_KEY"]
+    access_key_id:     ...,         # or ENV["AWS_ACCESS_KEY_ID"]
+    region:            "us-east-1", # or ENV["AWS_REGION"]
+    receive_message: {              # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
+      # wait_time_seconds: N,       # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
+      attribute_names: [
+        "ApproximateReceiveCount",
+        "SentTimestamp"
+      ]
+    }
+  }
+end
+```
+
+If you use Shoryuken with plain ruby worker class (not Rails), please call `configure_client` at the beginning of the worker file:
+
+```ruby
+Shoryuken.configure_client do |config|
+  config.aws = {
+    secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+    access_key_id:     ENV["AWS_ACCESS_KEY_ID"],
+    region:            ENV["AWS_REGION"]
+  }
+end
+
+class MyWorker
+end
+```
 
-The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
+The `aws` section is used to configure both the Aws objects used by Shoryuken internally, and also to set up some Shoryuken-specific config. The Shoryuken-specific keys are listed below, and you can expect any other key defined in that block to be passed on untouched to `Aws::SQS::Client#initialize`:
+
+- `account_id` is used when generating SNS ARNs
+- `sns_endpoint` can be used to explicitly override the SNS endpoint
+- `sqs_endpoint` can be used to explicitly override the SQS endpoint
+- `receive_message` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
+
+The `sns_endpoint` and `sqs_endpoint` Shoryuken-specific options will also fallback to the environment variables `AWS_SNS_ENDPOINT` and `AWS_SQS_ENDPOINT` respectively, if they are set.
 
 ### Configuration (producer side)
 
 'Producer' processes need permissions to put messages into SQS. There are a few ways:
 
+* Use the `configure_server` in Rails initializer
 * Ensure the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars are set.
 * Create a `~/.aws/credentials` file.
 * Set `Aws.config[:credentials]` from Ruby code (e.g. in a Rails initializer)
 * Use the Instance Profiles feature. The IAM role of the targeted machine must have an adequate SQS Policy.
 
+For example, use the `configure_server` in `config/initializers/shoryuken.rb`:
+
+```ruby
+Shoryuken.configure_client do |config|
+  config.aws = {
+    secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+    access_key_id:     ENV["AWS_ACCESS_KEY_ID"],
+    region:            ENV["AWS_REGION"]
+  }
+end
+
+Shoryuken.configure_server do |config|
+  config.aws = {
+    secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+    access_key_id:     ENV["AWS_ACCESS_KEY_ID"],
+    region:            ENV["AWS_REGION"]
+  }
+end
+```
+
+
 Note that storing your credentials into Amazon instances represents a security risk. Instance Profiles tends to be the best choice.
 
 You can read about these in more detail [here](http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html).

data/lib/shoryuken.rb

@@ -21,6 +21,7 @@ require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
 require 'shoryuken/sns_arn'
 require 'shoryuken/topic'
+require 'shoryuken/polling'
 
 module Shoryuken
   DEFAULTS = {
@@ -33,7 +34,8 @@ module Shoryuken
       startup: [],
       quiet: [],
       shutdown: [],
-    }
+    },
+    polling_strategy: Polling::WeightedRoundRobin,
   }
 
   @@queues = []

data/lib/shoryuken/environment_loader.rb

@@ -60,8 +60,8 @@ module Shoryuken
     end
 
     def initialize_logger
-      Shoryuken::Logging.initialize_logger(options[:logfile]) if options[:logfile]
-      Shoryuken.logger.level = Logger::DEBUG if options[:verbose]
+      Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
+      Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
     def load_rails

data/lib/shoryuken/fetcher.rb

@@ -1,26 +1,9 @@
 module Shoryuken
   class Fetcher
-    include Celluloid
     include Util
 
     FETCH_LIMIT = 10
 
-    def initialize(manager)
-      @manager = manager
-    end
-
-    def receive_messages(queue, limit)
-      # AWS limits the batch size by 10
-      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
-
-      options = (Shoryuken::AwsConfig.options[:receive_message] || {}).dup
-      options[:max_number_of_messages] = limit
-      options[:message_attribute_names] = %w(All)
-      options[:attribute_names] = %w(All)
-
-      Shoryuken::Client.queues(queue).receive_messages options
-    end
-
     def fetch(queue, available_processors)
       watchdog('Fetcher#fetch died') do
         started_at = Time.now
@@ -28,46 +11,34 @@ module Shoryuken
         logger.debug { "Looking for new messages in '#{queue}'" }
 
         begin
-          batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
-          limit = batch ? FETCH_LIMIT : available_processors
-
-          if (sqs_msgs = Array(receive_messages(queue, limit))).any?
-            logger.debug { "Found #{sqs_msgs.size} messages for '#{queue}'" }
-
-            if batch
-              @manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
-            else
-              sqs_msgs.each { |sqs_msg| @manager.async.assign(queue, sqs_msg) }
-            end
-
-            @manager.async.rebalance_queue_weight!(queue)
-          else
-            logger.debug { "No message found for '#{queue}'" }
-
-            @manager.async.pause_queue!(queue)
-          end
+          limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
 
+          sqs_msgs = Array(receive_messages(queue, limit))
+          logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" }
           logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
+          sqs_msgs
         rescue => ex
           logger.error { "Error fetching message: #{ex}" }
           logger.error { ex.backtrace.first }
+          []
         end
-
-        @manager.async.dispatch
       end
-
     end
 
     private
 
-    def patch_sqs_msgs!(sqs_msgs)
-      sqs_msgs.instance_eval do
-        def message_id
-          "batch-with-#{size}-messages"
-        end
-      end
+    def receive_messages(queue, limit)
+      # AWS limits the batch size by 10
+      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
+
+      options = (Shoryuken.options[:aws][:receive_message] || {}).dup
+      options[:max_number_of_messages] = limit
+      options[:message_attribute_names] = %w(All)
+      options[:attribute_names] = %w(All)
+
+      options.merge!(queue.options)
 
-      sqs_msgs
+      Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -10,17 +10,16 @@ module Shoryuken
     def initialize
       @condvar = Celluloid::Condition.new
       @manager = Shoryuken::Manager.new_link(@condvar)
-      @fetcher = Shoryuken::Fetcher.new_link(manager)
 
       @done = false
 
-      manager.fetcher = @fetcher
+      manager.fetcher = Shoryuken::Fetcher.new
+      manager.polling_strategy = Shoryuken.options[:polling_strategy].new(Shoryuken.queues)
     end
 
     def stop(options = {})
       watchdog('Launcher#stop') do
         @done = true
-        @fetcher.terminate if @fetcher.alive?
 
         manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
         @condvar.wait

data/lib/shoryuken/manager.rb

@@ -7,9 +7,14 @@ module Shoryuken
     include Util
 
     attr_accessor :fetcher
+    attr_accessor :polling_strategy
+
+    exclusive :dispatch
 
     trap_exit :processor_died
 
+    BATCH_LIMIT = 10
+
     def initialize(condvar)
       @count = Shoryuken.options[:concurrency] || 25
       raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
@@ -18,9 +23,9 @@ module Shoryuken
 
       @done = false
 
-      @busy  = []
-      @ready = @count.times.map { build_processor }
-      @threads = {}
+      @busy_processors  = []
+      @busy_threads = {}
+      @ready_processors = @count.times.map { build_processor }
     end
 
     def start
@@ -40,16 +45,14 @@ module Shoryuken
 
         fire_event(:shutdown, true)
 
-        @fetcher.terminate if @fetcher.alive?
+        logger.info { "Shutting down #{@ready_processors.size} quiet workers" }
 
-        logger.info { "Shutting down #{@ready.size} quiet workers" }
-
-        @ready.each do |processor|
+        @ready_processors.each do |processor|
           processor.terminate if processor.alive?
         end
-        @ready.clear
+        @ready_processors.clear
 
-        return after(0) { @finished.signal } if @busy.empty?
+        return after(0) { @finished.signal } if @busy_processors.empty?
 
         if options[:shutdown]
           hard_shutdown_in(options[:timeout])
@@ -63,14 +66,15 @@ module Shoryuken
       watchdog('Manager#processor_done died') do
         logger.debug { "Process done for '#{queue}'" }
 
-        @threads.delete(processor.object_id)
-        @busy.delete processor
+        @busy_processors.delete(processor)
+        @busy_threads.delete(processor.object_id)
 
         if stopped?
           processor.terminate if processor.alive?
-          return after(0) { @finished.signal } if @busy.empty?
+          return after(0) { @finished.signal } if @busy_processors.empty?
         else
-          @ready << processor
+          @ready_processors << processor
+          async.dispatch
         end
       end
     end
@@ -79,13 +83,14 @@ module Shoryuken
       watchdog("Manager#processor_died died") do
         logger.error { "Process died, reason: #{reason}" }
 
-        @threads.delete(processor.object_id)
-        @busy.delete processor
+        @busy_processors.delete(processor)
+        @busy_threads.delete(processor.object_id)
 
         if stopped?
-          return after(0) { @finished.signal } if @busy.empty?
+          return after(0) { @finished.signal } if @busy_processors.empty?
         else
-          @ready << build_processor
+          @ready_processors << build_processor
+          async.dispatch
         end
       end
     end
@@ -94,61 +99,27 @@ module Shoryuken
       @done
     end
 
-    def assign(queue, sqs_msg)
-      watchdog('Manager#assign died') do
-        logger.debug { "Assigning #{sqs_msg.message_id}" }
-
-        processor = @ready.pop
-        @busy << processor
-
-        processor.async.process(queue, sqs_msg)
-      end
-    end
-
-    def rebalance_queue_weight!(queue)
-      watchdog('Manager#rebalance_queue_weight! died') do
-        if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
-          logger.info { "Increasing '#{queue}' weight to #{current + 1}, max: #{original}" }
-
-          @queues << queue
-        end
-      end
-    end
-
-    def pause_queue!(queue)
-      return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
-
-      logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty" }
-
-      @queues.delete(queue)
-
-      after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
-    end
-
-
     def dispatch
       return if stopped?
 
-      logger.debug { "Ready: #{@ready.size}, Busy: #{@busy.size}, Active Queues: #{unparse_queues(@queues)}" }
+      logger.debug { "Ready: #{@ready_processors.size}, Busy: #{@busy_processors.size}, Active Queues: #{polling_strategy.active_queues}" }
 
-      if @ready.empty?
+      if @ready_processors.empty?
         logger.debug { 'Pausing fetcher, because all processors are busy' }
-
         dispatch_later
         return
       end
 
-      if (queue = next_queue)
-        @fetcher.async.fetch(queue, @ready.size)
-      else
+      queue = polling_strategy.next_queue
+      if queue.nil?
         logger.debug { 'Pausing fetcher, because all queues are paused' }
-
-        @fetcher_paused = true
+        dispatch_later
+        return
       end
-    end
 
-    def real_thread(proxy_id, thr)
-      @threads[proxy_id] = thr
+      batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
+
+      async.dispatch
     end
 
     private
@@ -160,67 +131,48 @@ module Shoryuken
       end
     end
 
-    def build_processor
-      processor = Processor.new_link(current_actor)
-      processor.proxy_id = processor.object_id
-      processor
-    end
-
-    def restart_queue!(queue)
-      return if stopped?
-
-      unless @queues.include? queue
-        logger.debug { "Restarting '#{queue}'" }
-
-        @queues << queue
-
-        if @fetcher_paused
-          logger.debug { 'Restarting fetcher' }
+    def assign(queue, sqs_msg)
+      watchdog('Manager#assign died') do
+        logger.debug { "Assigning #{sqs_msg.message_id}" }
 
-          @fetcher_paused = false
+        processor = @ready_processors.pop
+        @busy_threads[processor.object_id] = processor.running_thread
+        @busy_processors << processor
 
-          dispatch
-        end
+        processor.async.process(queue, sqs_msg)
       end
     end
 
-    def current_queue_weight(queue)
-      queue_weight(@queues, queue)
+    def dispatch_batch(queue)
+      batch = fetcher.fetch(queue, BATCH_LIMIT)
+      polling_strategy.messages_found(queue.name, batch.size)
+      assign(queue.name, patch_batch!(batch))
     end
 
-    def original_queue_weight(queue)
-      queue_weight(Shoryuken.queues, queue)
+    def dispatch_single_messages(queue)
+      messages = fetcher.fetch(queue, @ready_processors.size)
+      polling_strategy.messages_found(queue.name, messages.size)
+      messages.each { |message| assign(queue.name, message) }
     end
 
-    def queue_weight(queues, queue)
-      queues.count { |q| q == queue }
+    def batched_queue?(queue)
+      Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
-    def next_queue
-      return nil if @queues.empty?
-
-      # get/remove the first queue in the list
-      queue = @queues.shift
-
-      unless defined?(::ActiveJob) || !Shoryuken.worker_registry.workers(queue).empty?
-        # when no worker registered pause the queue to avoid endless recursion
-        logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered" }
-
-        after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
-
-        return next_queue
-      end
-
-      # add queue back to the end of the list
-      @queues << queue
+    def delay
+      Shoryuken.options[:delay].to_f
+    end
 
-      queue
+    def build_processor
+      processor = Processor.new_link(current_actor)
+      processor.proxy_id = processor.object_id
+      processor
     end
 
     def soft_shutdown(delay)
-      logger.info { "Waiting for #{@busy.size} busy workers" }
+      logger.info { "Waiting for #{@busy_processors.size} busy workers" }
 
-      if @busy.size > 0
+      if @busy_processors.size > 0
         after(delay) { soft_shutdown(delay) }
       else
         @finished.signal
@@ -228,16 +180,16 @@ module Shoryuken
     end
 
     def hard_shutdown_in(delay)
-      logger.info { "Waiting for #{@busy.size} busy workers" }
+      logger.info { "Waiting for #{@busy_processors.size} busy workers" }
       logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
 
       after(delay) do
         watchdog('Manager#hard_shutdown_in died') do
-          if @busy.size > 0
-            logger.info { "Hard shutting down #{@busy.size} busy workers" }
+          if @busy_processors.size > 0
+            logger.info { "Hard shutting down #{@busy_processors.size} busy workers" }
 
-            @busy.each do |processor|
-              if processor.alive? && t = @threads.delete(processor.object_id)
+            @busy_processors.each do |processor|
+              if processor.alive? && t = @busy_threads.delete(processor.object_id)
                 t.raise Shutdown
               end
             end
@@ -247,5 +199,15 @@ module Shoryuken
         end
       end
     end
+
+    def patch_batch!(sqs_msgs)
+      sqs_msgs.instance_eval do
+        def message_id
+          "batch-with-#{size}-messages"
+        end
+      end
+
+      sqs_msgs
+    end
   end
 end