sqewer 5.0.8 → 5.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 303e0674eeb56dc7ff4a8056482a828988eb53c5
-  data.tar.gz: 47762bf5591cac2d28a6a6b7078a97c82a697b45
+  metadata.gz: 84caf342300d2128e0e128efe4d235413bf011bc
+  data.tar.gz: f695a18f5e6b94a5c1854e4a5eb973ab7066def1
 SHA512:
-  metadata.gz: fa6a9832e1833e9ed3ec3b2a316001b4637bc9eeec997474a265aa9242c5b3e490752fa8c12f4d238a5bd6c2ef8a8012ca2e37651d853d4f53b879fc7e253a9e
-  data.tar.gz: 10327bbbe1e13113f7ed4910c6feb90f29b1148ee0af9f98762eab127287bfa3038be7609709cd76079efea10038b5d06c25ae9c851c796bfc479947dea33892
+  metadata.gz: 408c64f831f9441ed93609b07f037dff99de6af2d5ece6daebc95bf604c43a6518e2a9dc18a5c8ec37cdfe5d138f0d1a1b3a82b9b1a6554f4b34832992d275b0
+  data.tar.gz: 9b1c80ccadffe1b7f006a3310272d9fbdc715d3c45e22c1fcf8ace0d9bc8e2d300ac27096a745af0e80d031b41b454f60c46c6830263542c512cb7382312cd4d

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 2017-06-16
+- Released v5.0.9 to Rubygems.org.
+- Testing with fake_sqs when the daemon was not running or with a misconfigured SQS_QUEUE_URL could lead to Sqewer seemingly hanging. The actual cause was a _very_ large amount of retries were being performed. The amount of retries has been adjusted to a more reasonable number.
+- An exception in the message fetching thread could lead to the receiving thread silently dying, while leaving the worker threads running without any work to do. Uncaught exceptions in the receiving thread now lead to a graceful shutdown of the worker.
+
 ### 2017-06-12
 - Released v5.0.8 to Rubygems.org.
 - Retry sending and deleting messages when `sender_fault=false`.

data/lib/sqewer/connection.rb

@@ -9,6 +9,7 @@ class Sqewer::Connection
   DEFAULT_TIMEOUT_SECONDS = 5
   BATCH_RECEIVE_SIZE = 10
   MAX_RANDOM_FAILURES_PER_CALL = 10
+  MAX_RANDOM_RECEIVE_FAILURES = 100 # sure to hit the max_elapsed_time of 900 seconds
 
   NotOurFaultAwsError = Class.new(StandardError)
 
@@ -39,14 +40,17 @@ class Sqewer::Connection
     @queue_url = queue_url
   end
 
-  # Receive at most 10 messages from the queue, and return the array of Message objects.
+  # Receive at most 10 messages from the queue, and return the array of Message objects. Retries for at
+  # most 900 seconds (15 minutes) and then gives up, thereby crashing the read loop. If SQS is not available
+  # even after 15 minutes it is either down or the server is misconfigured. Either way it makes no sense to
+  # continue.
   #
   # @return [Array<Message>] an array of Message objects 
   def receive_messages
-    response = client.receive_message(queue_url: @queue_url,
-      wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: 10)
-    response.messages.map do | message |
-      Message.new(message.receipt_handle, message.body)
+    Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
+      response = client.receive_message(queue_url: @queue_url,
+        wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: BATCH_RECEIVE_SIZE)
+      response.messages.map {|message| Message.new(message.receipt_handle, message.body) }
     end
   end
 
@@ -125,7 +129,7 @@ class Sqewer::Connection
   private
 
   def handle_batch_with_retries(method, batch)
-    Retriable.retriable on: NotOurFaultAwsError, tries: MAX_RANDOM_FAILURES_PER_CALL do
+    Retriable.retriable on: [NotOurFaultAwsError, Seahorse::Client::NetworkingError], tries: MAX_RANDOM_FAILURES_PER_CALL do
       resp = client.send(method, queue_url: @queue_url, entries: batch)
       wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
       if wrong_messages.any?
@@ -139,27 +143,7 @@ class Sqewer::Connection
     end
   end
 
-  class RetryWrapper < Struct.new(:sqs_client)
-    MAX_RETRIES = 1000
-    # Provide retrying wrappers for all the methods of Aws::SQS::Client that we actually use
-    [:delete_message_batch, :send_message_batch, :receive_message].each do |retriable_method_name|
-      define_method(retriable_method_name) do |*args, **kwargs|
-        tries = 1
-        begin
-          sqs_client.public_send(retriable_method_name, *args, **kwargs)
-        rescue Seahorse::Client::NetworkingError => e
-          if (tries += 1) >= MAX_RETRIES
-            raise(e)
-          else
-            sleep 0.5
-            retry
-          end
-        end
-      end
-    end
-  end
-
   def client
-    @client ||= RetryWrapper.new(Aws::SQS::Client.new)
+    @client ||= Aws::SQS::Client.new
   end
 end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '5.0.8'
+  VERSION = '5.0.9'
 end

data/lib/sqewer/worker.rb

@@ -33,6 +33,9 @@ class Sqewer::Worker
   # @return [Fixnum] the number of worker threads set up for this Worker
   attr_reader :num_threads
 
+  # @return [Symbol] the current state of this Worker
+  attr_reader :state
+
   # Returns a Worker instance, configured based on the default components
   #
   # @return [Sqewer::Worker]
@@ -94,22 +97,28 @@ class Sqewer::Worker
 
     # Create the provider thread. When the execution queue is exhausted,
     # grab new messages and place them on the local queue.
+    owning_worker = self # self won't be self anymore in the thread
     provider = Thread.new do
       loop do
-        break if stopping?
-
-        if queue_has_capacity?
-          messages = @connection.receive_messages
-          if messages.any?
-            messages.each {|m| @execution_queue << m }
-            @logger.debug { "[worker] Received and buffered %d messages" % messages.length } if messages.any?
+        begin
+          break if stopping?
+
+          if queue_has_capacity?
+            messages = @connection.receive_messages
+            if messages.any?
+              messages.each {|m| @execution_queue << m }
+              @logger.debug { "[worker] Received and buffered %d messages" % messages.length } if messages.any?
+            else
+              @logger.debug { "[worker] No messages received" }
+              Thread.pass
+            end
           else
-            @logger.debug { "[worker] No messages received" }
-            Thread.pass
+            @logger.debug { "[worker] Cache is full (%d items), postponing receive" % @execution_queue.length }
+            sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
           end
-        else
-          @logger.debug { "[worker] Cache is full (%d items), postponing receive" % @execution_queue.length }
-          sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+        rescue StandardError => e
+          @logger.fatal "Exiting because message receiving thread died. Exception causing this: #{e.inspect}"
+          owning_worker.stop # allow any queues and/or running jobs to complete
         end
       end
     end
@@ -158,7 +167,7 @@ class Sqewer::Worker
     true
   end
 
-  # Peforms a hard shutdown by killing all the threads
+  # Performs a hard shutdown by killing all the threads
   def kill
     @state.transition! :stopping
     @logger.info { '[worker] Killing (unclean shutdown), will kill all threads'}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 5.0.8
+  version: 5.0.9
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-12 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk