shoryuken 3.0.7 → 3.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d0347fa63667fbc483984759877ee74a65919cd
-  data.tar.gz: a06ea715ee035e398cdab792c2647b3e6575f216
+  metadata.gz: 3748f41972ac2a8c233278b33432cb5c2056848a
+  data.tar.gz: fa952b333c483077908c53bd0515c49026fd4d33
 SHA512:
-  metadata.gz: 4a560a9dbc3f31706263ac0f3d15a90b976e7dcc2b040eab0317e0ceb782ea239cea95c8e96d0c6bf0ee7cc4a3d4be30fb2249ddc177c35500a60b463e52beee
-  data.tar.gz: 94776b105d1288009d3b5d93022a6c0fd95a5981daa13925c0453802d90e6215e0d37eb5b044af8ce267559b9becdc49921e9842e6b34b672cb892762f6e4615
+  metadata.gz: 9cd990370ec8fc9da9f46400e98076eeea77c0dd4cb9ae1c291319ba0ed496ea9c536f6cdcb91757853f05c68a3e71d7f681a7838a077746a932d5b0c7a7560c
+  data.tar.gz: 3dc1ac1e696112e865bc69fce0bc3eef745658de4627199f830fde4ac4d2351c2792629d51210d7bd92026b61dfe72b715073a14d5f3c9b7da93e87cf2486520

data/.rubocop.yml

@@ -32,7 +32,7 @@ Metrics/ParameterLists:
   Enabled: false
 
 Metrics/LineLength:
-  Max: 120
+  Max: 130
 
 Metrics/MethodLength :
   Enabled: false

data/CHANGELOG.md

@@ -1,4 +1,16 @@
+## [v3.0.8] - 2017-06-02
+
+- Fix miss handling empty batch fetches
+ - [#376](https://github.com/phstc/shoryuken/pull/376)
+
+- Various minor styling changes :lipstick:
+ - [#373](https://github.com/phstc/shoryuken/pull/373)
+
+- Logout when batch delete returns any failure
+ - [#371](https://github.com/phstc/shoryuken/pull/371)
+
 ## [v3.0.7] - 2017-05-18
+
 - Trigger events for dispatch
  - [#362](https://github.com/phstc/shoryuken/pull/362)
 
@@ -9,10 +21,12 @@
  - [#366](https://github.com/phstc/shoryuken/pull/366)
 
 ## [v3.0.6] - 2017-04-11
+
 - Fix delay option type
  - [#356](https://github.com/phstc/shoryuken/pull/356)
 
 ## [v3.0.5] - 2017-04-09
+
 - Pause endless dispatcher to avoid CPU overload
  - [#354](https://github.com/phstc/shoryuken/pull/354)
 
@@ -24,7 +38,9 @@
 
 - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
  - [#344](https://github.com/phstc/shoryuken/pull/344)
+
 ## [v3.0.4] - 2017-03-24
+
 - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
  - [#344](https://github.com/phstc/shoryuken/pull/344)
 
@@ -32,18 +48,22 @@
  - [#345](https://github.com/phstc/shoryuken/pull/345)
 
 ## [v3.0.3] - 2017-03-19
+
 - Update `sqs` CLI commands to use `get_queue_url` when appropriated
  - [#341](https://github.com/phstc/shoryuken/pull/341)
 
 ## [v3.0.2] - 2017-03-19
+
 - Fix custom SQS client initialization
  - [#335](https://github.com/phstc/shoryuken/pull/335)
 
 ## [v3.0.1] - 2017-03-13
+
 - Fix commands sqs mv and dump `options.delete` checker
  - [#332](https://github.com/phstc/shoryuken/pull/332)
 
 ## [v3.0.0] - 2017-03-12
+
 - Replace Celluloid with Concurrent Ruby
  - [#291](https://github.com/phstc/shoryuken/pull/291)
 
@@ -63,6 +83,7 @@
  - [#330](https://github.com/phstc/shoryuken/pull/330)
 
 ## [v2.1.3] - 2017-01-27
+
 - Show a warn message when batch isn't supported
  - [#302](https://github.com/phstc/shoryuken/pull/302)
 
@@ -73,6 +94,7 @@
  - [#307](https://github.com/phstc/shoryuken/pull/307)
 
 ## [v2.1.2] - 2016-12-22
+
 - Fix loading `logfile` from shoryuken.yml
  - [#296](https://github.com/phstc/shoryuken/pull/296)
 
@@ -92,10 +114,12 @@
  - [#284](https://github.com/phstc/shoryuken/pull/284)
 
 ## [v2.1.1] - 2016-12-05
+
 - Fix aws deprecation warning message
  - [#279](https://github.com/phstc/shoryuken/pull/279)
 
 ## [v2.1.0] - 2016-12-03
+
 - Fix celluloid "running in BACKPORTED mode" warning
  - [#260](https://github.com/phstc/shoryuken/pull/260)
 

data/bin/cli/sqs.rb

@@ -32,7 +32,10 @@ module Shoryuken
               queue_url: url,
               entries: batch.map { |message| { id: message.message_id, receipt_handle: message.receipt_handle } }
             ).failed.any? do |failure|
-              say "Could not delete #{failure.id}, code: #{failure.code}", :yellow
+              say(
+                "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}",
+                :yellow
+              )
             end
           end
         end

data/examples/default_worker.rb

@@ -4,6 +4,6 @@ class DefaultWorker
   shoryuken_options queue: 'default', auto_delete: true
 
   def perform(sqs_msg, body)
-    Shoryuken.logger.debug("Received message: '#{body}'")
+    Shoryuken.logger.debug("Received message: #{body}")
   end
 end

data/lib/shoryuken/environment_loader.rb

@@ -42,7 +42,7 @@ module Shoryuken
     def config_file_options
       return {} unless (path = options[:config_file])
 
-      fail ArgumentError, "The supplied config file '#{path}' does not exist" unless File.exist?(path)
+      fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
       YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
     end
@@ -141,7 +141,7 @@ module Shoryuken
       queues_with_workers = Shoryuken.worker_registry.queues
 
       (all_queues - queues_with_workers).each do |queue|
-        Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
+        Shoryuken.logger.warn { "No worker supplied for #{queue}" }
       end
     end
   end

data/lib/shoryuken/fetcher.rb

@@ -7,18 +7,18 @@ module Shoryuken
     def fetch(queue, available_processors)
       started_at = Time.now
 
-      logger.debug { "Looking for new messages in '#{queue}'" }
+      logger.debug { "Looking for new messages in #{queue}" }
 
       begin
         limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
 
         sqs_msgs = Array(receive_messages(queue, limit))
-        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
-        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
+        logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
         sqs_msgs
       rescue => ex
-        logger.error { "Error fetching message: #{ex}" }
-        logger.error { ex.backtrace.first }
+        logger.error { "Error fetching message: #{ex.message}" }
+        logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
         []
       end
     end

data/lib/shoryuken/manager.rb

@@ -32,7 +32,7 @@ module Shoryuken
       @done.make_true
 
       if (callback = Shoryuken.stop_callback)
-        logger.info { 'Calling Shoryuken.on_stop block' }
+        logger.info { 'Calling on_stop callback' }
         callback.call
       end
 
@@ -50,12 +50,12 @@ module Shoryuken
     end
 
     def processor_failed(ex)
-      logger.error ex
-      logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+      logger.error { "Processor failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
     end
 
     def processor_done(queue)
-      logger.debug { "Process done for '#{queue}'" }
+      logger.debug { "Process done for #{queue}" }
     end
 
     private
@@ -98,7 +98,7 @@ module Shoryuken
     end
 
     def dispatch_batch(queue)
-      batch = @fetcher.fetch(queue, BATCH_LIMIT)
+      return if (batch = @fetcher.fetch(queue, BATCH_LIMIT)).none?
       @polling_strategy.messages_found(queue.name, batch.size)
       assign(queue.name, patch_batch!(batch))
     end

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -5,18 +5,13 @@ module Shoryuken
         def call(worker, queue, sqs_msg, body)
           yield
 
-          auto_delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
+          return unless worker.class.auto_delete?
 
-          if auto_delete
-            entries = [sqs_msg].flatten.map.with_index do |message, i|
-              { id: i.to_s, receipt_handle: message.receipt_handle }
-            end
+          entries = [sqs_msg].flatten.map.with_index { |message, i| { id: i.to_s, receipt_handle: message.receipt_handle } }
 
-            Shoryuken::Client.queues(queue).delete_messages(entries: entries)
-          end
+          Shoryuken::Client.queues(queue).delete_messages(entries: entries)
         end
       end
     end
   end
 end
-

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -7,6 +7,8 @@ module Shoryuken
         EXTEND_UPFRONT_SECONDS = 5
 
         def call(worker, queue, sqs_msg, body)
+          return yield unless worker.class.auto_visibility_timeout?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Auto extend visibility isn't supported for batch workers" }
             return yield
@@ -34,11 +36,11 @@ module Shoryuken
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => e
+              rescue => ex
                 logger.error do
                   'Could not auto extend the message ' \
                   "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{e.message}"
+                  "visibility timeout. Error: #{ex.message}"
                 end
               end
             end
@@ -46,8 +48,6 @@ module Shoryuken
         end
 
         def auto_visibility_timer(worker, queue, sqs_msg, body)
-          return unless worker.class.auto_visibility_timeout?
-
           MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end
       end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -5,6 +5,8 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
+          return yield unless worker.class.exponential_backoff?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Exponential backoff isn't supported for batch workers" }
             return yield
@@ -23,7 +25,7 @@ module Shoryuken
 
           logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
           # since we didn't raise, lets log the backtrace for debugging purposes.
-          logger.debug ex.backtrace.join("\n") unless ex.backtrace.nil?
+          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
         end
 
         private
@@ -51,7 +53,7 @@ module Shoryuken
 
           sqs_msg.change_visibility(visibility_timeout: next_visibility_timeout(interval.to_i, started_at))
 
-          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds." }
+          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds" }
 
           true
         end

data/lib/shoryuken/middleware/server/timing.rb

@@ -20,9 +20,9 @@ module Shoryuken
               end
 
               logger.info { "completed in: #{total_time} ms" }
-            rescue => e
+            rescue
               logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise e
+              raise
             end
           end
         end

data/lib/shoryuken/polling.rb

@@ -89,7 +89,7 @@ module Shoryuken
         maximum_weight = maximum_queue_weight(queue)
         current_weight = current_queue_weight(queue)
         if maximum_weight > current_weight
-          logger.info { "Increasing '#{queue}' weight to #{current_weight + 1}, max: #{maximum_weight}" }
+          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
           @queues << queue
         end
       end
@@ -103,7 +103,7 @@ module Shoryuken
       def pause(queue)
         return unless @queues.delete(queue)
         @paused_queues << [Time.now + delay, queue]
-        logger.debug "Paused '#{queue}'"
+        logger.debug "Paused #{queue}"
       end
 
       def unpause_queues
@@ -111,7 +111,7 @@ module Shoryuken
         return if Time.now < @paused_queues.first[0]
         pause = @paused_queues.shift
         @queues << pause[1]
-        logger.debug "Unpaused '#{pause[1]}'"
+        logger.debug "Unpaused #{pause[1]}"
       end
 
       def current_queue_weight(queue)
@@ -197,7 +197,7 @@ module Shoryuken
       def pause(queue)
         return unless delay > 0
         @paused_until[queue] = Time.now + delay
-        logger.debug "Paused '#{queue}'"
+        logger.debug "Paused #{queue}"
       end
     end
   end

data/lib/shoryuken/processor.rb

@@ -50,8 +50,8 @@ module Shoryuken
           body_parser.load(sqs_msg.body)
         end
       end
-    rescue => e
-      logger.error { "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
+    rescue => ex
+      logger.error { "Error parsing the message body: #{ex.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
       raise
     end
   end

data/lib/shoryuken/queue.rb

@@ -1,5 +1,7 @@
 module Shoryuken
   class Queue
+    include Util
+
     FIFO_ATTR               = 'FifoQueue'
     MESSAGE_GROUP_ID        = 'ShoryukenMessage'
     VISIBILITY_TIMEOUT_ATTR = 'VisibilityTimeout'
@@ -10,8 +12,8 @@ module Shoryuken
       self.name   = name
       self.client = client
       self.url    = client.get_queue_url(queue_name: name).queue_url
-    rescue Aws::SQS::Errors::NonExistentQueue => e
-      raise e, "The specified queue '#{name}' does not exist."
+    rescue Aws::SQS::Errors::NonExistentQueue => ex
+      raise ex, "The specified queue #{name} does not exist."
     end
 
     def visibility_timeout
@@ -19,7 +21,13 @@ module Shoryuken
     end
 
     def delete_messages(options)
-      client.delete_message_batch(options.merge(queue_url: url))
+      client.delete_message_batch(
+        options.merge(queue_url: url)
+      ).failed.any? do |failure|
+        logger.error do
+          "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
+        end
+      end
     end
 
     def send_message(options)

data/lib/shoryuken/runner.rb

@@ -44,7 +44,7 @@ module Shoryuken
       @launcher = Shoryuken::Launcher.new
 
       if (callback = Shoryuken.start_callback)
-        logger.info { 'Calling Shoryuken.on_start block' }
+        logger.info { 'Calling on_start callback' }
         callback.call
       end
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.0.7'.freeze
+  VERSION = '3.0.8'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -48,6 +48,14 @@ module Shoryuken
         !!get_shoryuken_options['auto_visibility_timeout']
       end
 
+      def exponential_backoff?
+        !!get_shoryuken_options['retry_intervals']
+      end
+
+      def auto_delete?
+        !!(get_shoryuken_options['delete'] || get_shoryuken_options['auto_delete'])
+      end
+
       def get_shoryuken_options # :nodoc:
         @shoryuken_options || Shoryuken.default_worker_options
       end

data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
 
   context 'when batch worker' do
     it 'yields' do
-      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+      expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
     end
   end
 

data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
 
   context 'when batch worker' do
     it 'yields' do
-      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+      expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
     end
   end
 

data/spec/shoryuken/queue_spec.rb

@@ -4,19 +4,47 @@ describe Shoryuken::Queue do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name) { 'shoryuken' }
-  let(:queue_url) { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
+  let(:queue_url) { "https://eu-west-1.amazonaws.com:6059/0123456789/#{queue_name}" }
 
   subject { described_class.new(sqs, queue_name) }
-  before {
+
+  before do
     # Required as Aws::SQS::Client.get_queue_url returns 'String' when responses are stubbed,
     # which is not accepted by Aws::SQS::Client.get_queue_attributes for :queue_name parameter.
     allow(subject).to receive(:url).and_return(queue_url)
-  }
+  end
+
+  describe '#delete_messages' do
+    let(:entries) do
+      [
+        { id: '1', receipt_handle:  '1' },
+        { id: '2', receipt_handle:  '2' }
+      ]
+    end
+
+    it 'deletes' do
+      expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: []))
+
+      subject.delete_messages(entries: entries)
+    end
+
+    context 'when it fails' do
+      it 'logs the reason' do
+        failure = double(id: 'id', code: 'code', message: '...', sender_fault: false)
+        logger = double 'Logger'
+
+        expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: [failure]))
+        expect(subject).to receive(:logger).and_return(logger)
+        expect(logger).to receive(:error)
+
+        subject.delete_messages(entries: entries)
+      end
+    end
+  end
 
   describe '#send_message' do
-    before {
-      allow(subject).to receive(:fifo?).and_return(false)
-    }
+    before { allow(subject).to receive(:fifo?).and_return(false) }
+
     it 'accepts SQS request parameters' do
       # https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message-instance_method
       expect(sqs).to receive(:send_message).with(hash_including(message_body: 'msg1'))

data/spec/spec_helper.rb

@@ -5,6 +5,7 @@ require 'pry-byebug'
 require 'shoryuken'
 require 'json'
 require 'dotenv'
+require 'securerandom'
 Dotenv.load
 
 if ENV['CODECLIMATE_REPO_TOKEN']

data/test_workers/endless_uninterruptive_worker.rb

@@ -26,7 +26,7 @@ class EndlessUninterruptiveWorker
   end
 
   def perform(sqs_msg, body)
-    Shoryuken.logger.info("Received message: '#{body}'")
+    Shoryuken.logger.info("Received message: #{body}")
 
     execution_ms = self.class.random_number(self.class.max_execution_time)
     Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 3.0.7
+  version: 3.0.8
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-18 00:00:00.000000000 Z
+date: 2017-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler