sqewer 6.5.0 → 6.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a69e4c565c8d9532e871d74b077188f617fe513519fdc4f72f03ad25ba8c67d2
-  data.tar.gz: 92b73c32b4f9535fc53868f9ba61a3e579bfbba1244c191fc6c34c5a64bd7e2c
+  metadata.gz: cf1c2d6717fda50ce8a72721d00ffcfd2fae867f40f40a07a0cf48cbe94fe17b
+  data.tar.gz: fb1622a03a4f540aaa96d24dc3d5d39c5d400b658f371c88dc682844ac784ea5
 SHA512:
-  metadata.gz: 339e77231c3b905a5b3477c2bb933d5fadcbb3f6291f30f12b5e4682bc0d858ee9bb1b8da44e1ba59b46b4647cb3bb877b200f180fd78b91e2c775f0c7103755
-  data.tar.gz: 16eb06ee13263bd2af1ce18af5c8c4c5da20bf95bca329644a7ccce9fbbe77c54ea18769b3d2819e13f7f7c0374be293ec7beaa6f324ded588a1f2313591154b
+  metadata.gz: fd4e73838959531d646d0ed69fd8316e85075d3abe96b5362c58ff5de9af27b26cec142273bce674aaadf571275f1f91cf055c305be4b9797edd75eb89b74222
+  data.tar.gz: 9f4b9006f504bd4a0a85abea9d3c82bf2c70ccdfecf59095df58a1e6b1b8b3ed0b6ba819778f828c48e5dd1865c1f312c5feda78485e9ceab95565e9559cc870

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 6.5.1
+- Also retry on `Aws::SQS::Errors::InternalError` exception when receiving/sending messages. This will make
+  the receiving thread more resilient to sudden SQS failures. By the time SQS recovers the receiving thread
+  should stay alive.
+
 ### 6.5.0
 - Adds `$stdout.sync = true` to CLI to flush the logs to STDOUT
 

data/README.md

@@ -20,14 +20,14 @@ and to start processing, in your commandline handler:
 
     #!/usr/bin/env ruby
     require 'my_applicaion'
-    Sqewer::CLI.run
+    Sqewer::CLI.start
 
 To add arguments to the job
 
     class JobWithArgs
       include Sqewer::SimpleJob
       attr_accessor :times
-      
+
       def run
         ...
       end
@@ -88,6 +88,10 @@ Note that at this point only arguments that are raw JSON types are supported:
 
 If you need marshalable Ruby types there instead, you might need to implement a custom `Serializer.`
 
+### Sqewer::SimpleJob
+
+The module `Sqewer::SimpleJob` can be included to a job class to add some features, specially dealing with attributes, see more details [here](https://github.com/WeTransfer/sqewer/blob/master/lib/sqewer/simple_job.rb).
+
 ## Jobs spawning dependent jobs
 
 If your `run` method on the job object accepts arguments (has non-zero `arity` ) the `ExecutionContext` will
@@ -119,11 +123,11 @@ include all the keyword arguments needed to instantiate the job when executing.
       def initialize(to:, body:)
         ...
       end
-      
+
       def run()
         ...
       end
-      
+
       def to_h
         {to: @to, body: @body}
       end
@@ -151,7 +155,7 @@ conform to the job serialization format used internally. For example, you can ha
         message = JSON.load(message_blob)
         return if message['Service'] # AWS test
         return HandleS3Notification.new(message) if message['Records']
-        
+
         super # as default
       end
     end
@@ -176,7 +180,7 @@ The very minimal executable for running jobs would be this:
 
     #!/usr/bin/env ruby
     require 'my_applicaion'
-    Sqewer::CLI.run
+    Sqewer::CLI.start
 
 This will connect to the queue at the URL set in the `SQS_QUEUE_URL` environment variable, and
 use all the default parameters. The `CLI` module will also set up a signal handler to terminate
@@ -233,9 +237,9 @@ you generate. For example, you could use a pipe. But in a more general case some
           ActiveRAMGobbler.fetch_stupendously_many_things.each do |...|
           end
         end
-        
+
         _, status = Process.wait2(pid)
-        
+
         # Raise an error in the parent process to signal Sqewer that the job failed
         # if the child exited with a non-0 status
         raise "Child process crashed" unless status.exitstatus && status.exitstatus.zero?
@@ -252,7 +256,7 @@ You can wrap job processing in middleware. A full-featured middleware class look
         # msg_id is the receipt handle, msg_payload is the message body string, msg_attributes are the message's attributes
         yield
       end
-      
+
       # Surrounds the actual job execution
       def around_execution(job, context)
         # job is the actual job you will be running, context is the ExecutionContext.
@@ -378,7 +382,7 @@ products using this library, was a very bad idea (more workload for deployment).
 
 ## Why so many configurable components?
 
-Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your 
+Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
 implementation in instead. One product needs foreign-submitted SQS jobs (S3 notifications). Another product
 needs a custom Logger subclass. Yet another product needs process-based concurrency on top of threads.
 Yet another process needs to manage database connections when running the jobs. Have 3-4 of those, and a
@@ -398,7 +402,7 @@ Because I found that a producer-consumer model with a thread pool works quite we
 the Ruby standard library alone.
 
 ## Contributing to the library
- 
+
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
 * Fork the project.

data/lib/sqewer/connection.rb

@@ -52,9 +52,9 @@ class Sqewer::Connection
   # even after 15 minutes it is either down or the server is misconfigured. Either way it makes no sense to
   # continue.
   #
-  # @return [Array<Message>] an array of Message objects 
+  # @return [Array<Message>] an array of Message objects
   def receive_messages
-    Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
+    Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_RECEIVE_FAILURES do
       response = client.receive_message(
         queue_url: @queue_url,
         attribute_names: ['All'],
@@ -69,7 +69,7 @@ class Sqewer::Connection
   #
   # @param message_body[String] the message to send
   # @param kwargs_for_send[Hash] additional arguments for the submit (such as `delay_seconds`).
-  # Passes the arguments to the AWS SDK. 
+  # Passes the arguments to the AWS SDK.
   # @return [void]
   def send_message(message_body, **kwargs_for_send)
     send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
@@ -191,8 +191,12 @@ class Sqewer::Connection
 
   private
 
+  def network_and_aws_sdk_errors
+    [NotOurFaultAwsError, Seahorse::Client::NetworkingError, Aws::SQS::Errors::InternalError]
+  end
+
   def handle_batch_with_retries(method, batch)
-    Retriable.retriable on: [NotOurFaultAwsError, Seahorse::Client::NetworkingError], tries: MAX_RANDOM_FAILURES_PER_CALL do
+    Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_FAILURES_PER_CALL do
       resp = client.send(method, queue_url: @queue_url, entries: batch)
       wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
       if wrong_messages.any?

data/lib/sqewer/simple_job.rb

@@ -4,6 +4,13 @@
 # * initialize() will have keyword access to all accessors, and will ensure you have called each one of them
 # * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
 # * inspect() will provide a sensible default string representation for logging
+#
+# This module validates if the attributes defined in the job class are the same as
+# those persisted in the queue. More details on `Sqewer::SimpleJob#initialize`.
+# Because of this, it's required to create a new job class when adding or removing
+# an attribute.
+# This mechanism guarantees strong consistency. Without it, a new deployed job class
+# could process old incompatible payloads.
 module Sqewer::SimpleJob
   UnknownJobAttribute = Class.new(Sqewer::Error)
   MissingAttribute = Class.new(Sqewer::Error)
@@ -53,7 +60,7 @@ module Sqewer::SimpleJob
       accessor = "#{k}="
       touched_attributes << k
       unless respond_to?(accessor)
-        raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}" 
+        raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
       end
 
       send("#{k}=", v)

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '6.5.0'
+  VERSION = '6.5.1'
 end

data/lib/sqewer/worker.rb

@@ -91,7 +91,7 @@ class Sqewer::Worker
 
     consumers = (1..@num_threads).map do
       Thread.new do
-        catch(:goodbye) { loop {take_and_execute} }
+        loop { take_and_execute }
       end
     end
 
@@ -233,8 +233,11 @@ class Sqewer::Worker
     message = @execution_queue.pop(nonblock=true)
     handle_message(message)
   rescue ThreadError # Queue is empty
-    throw :goodbye if stopping?
-    sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+    if stopping?
+      Thread.current.exit
+    else
+      sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+    end
   rescue => e # anything else, at or below StandardError that does not need us to quit
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..64], e.class, e.message] }
     e.backtrace.each { |s| @logger.debug{"\t#{s}"} }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 6.5.0
+  version: 6.5.1
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-25 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -285,7 +285,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Process jobs from SQS