sqewer 6.4.0 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0251b1700672c6dc8722779c0865d3757aff9de9ea747d4739863521931cc0c
-  data.tar.gz: 0533f556c4cad614cf7973818126996c8a8bea51d1791ca22df059ab65a5202a
+  metadata.gz: 7bef1bf31a141c142252a3ff3a898351808d378f0538197769091fabcfdf7e0e
+  data.tar.gz: 6eeffee2b8ec4009507ad192aede407c5ab0bb929e694f89f7bf2394ea3f3301
 SHA512:
-  metadata.gz: 7d9d8f211697dc585517fd6c431e32f7500711888a3b97cf94aa7d4afbd043da2790407242a81a1ba37bca905cf6b454f73ee98a5015550768c948fe53eaa315
-  data.tar.gz: 38369a4c3a74e3bdb5616705744ad732a363757c3eeadb04dde327f77d9251633047851e446454a0126895acbaf194d44081c3afd0192a509992d592e164a2a5
+  metadata.gz: 1b7a6faf35f31cc8fa8f33d063964553308cfef543a2e35c26dccfc55e9819a7754eaff9a77c35233d6e230269f2adc06c8ee23902fda5f41e8c4d0708b6ff02
+  data.tar.gz: 9aeedcf29cabcb941037df8c51712d908901ab7700c75649ca40986418fc7462ef17218a7431d798cdf43da6ba545b8e3d87bc56735c34e9dac7181579f63560

data/.travis.yml

@@ -1,11 +1,7 @@
 gemfile:
-  - gemfiles/Gemfile.rails-4.2.x
   - gemfiles/Gemfile.rails-5.0.x
   - gemfiles/Gemfile.rails-5.1.x
 rvm:
-  - 2.3.7
-  - 2.4.5
-  - 2.5.1
   - 2.6.5
   - 2.7.0
 cache: bundler
@@ -13,15 +9,3 @@ sudo: false
 env:
   global:
   - AWS_REGION=eu-central-1
-matrix:
-  exclude:
-  - rvm: 2.3.7
-    gemfile: gemfiles/Gemfile.rails-5.0.x
-  - rvm: 2.3.7
-    gemfile: gemfiles/Gemfile.rails-5.1.x
-  - rvm: 2.5.1
-    gemfile: gemfiles/Gemfile.rails-4.2.x
-  - rvm: 2.6.5
-    gemfile: gemfiles/Gemfile.rails-4.2.x
-  - rvm: 2.7.0
-    gemfile: gemfiles/Gemfile.rails-4.2.x

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+### 8.0.0
+- Remove method `Sqewer.client=`
+- Change `Sqewer::Connection` to use a singleton SQS client by default
+
+### 7.0.0
+- Remove support of Ruby 2.3, 2.4 and 2.5
+- Remove support of Rails 4
+- Change `Sqewer::Connection` to preferentially use a singleton instance of `Aws::SQS::Client`, which can be set using `Sqewer.client=`. This avoids many HTTP requests to the AWS metadata endpoint when getting credentials.
+
+### 6.5.1
+- Also retry on `Aws::SQS::Errors::InternalError` exception when receiving/sending messages. This will make
+  the receiving thread more resilient to sudden SQS failures. By the time SQS recovers the receiving thread
+  should stay alive.
+
+### 6.5.0
+- Adds `$stdout.sync = true` to CLI to flush the logs to STDOUT
+
+### 6.4.1
+- Retrieve attributes when receiving messages from SQS
+
 ### 6.4.0
 - Raise an exception in submit! if the job serializes to a message that is
   above the native SQS limit for message size.

data/README.md

@@ -20,14 +20,14 @@ and to start processing, in your commandline handler:
 
     #!/usr/bin/env ruby
     require 'my_applicaion'
-    Sqewer::CLI.run
+    Sqewer::CLI.start
 
 To add arguments to the job
 
     class JobWithArgs
       include Sqewer::SimpleJob
       attr_accessor :times
-      
+
       def run
         ...
       end
@@ -48,7 +48,7 @@ The messages will only be deleted from SQS once the job execution completes with
 
 ## Requirements
 
-Ruby 2.1+, version 2 of the AWS SDK. You can also run Sqewer backed by a SQLite database file, which can be handy for development situations.
+Ruby 2.6+, version 2 of the AWS SDK. You can also run Sqewer backed by a SQLite database file, which can be handy for development situations.
 
 ## Job storage
 
@@ -88,6 +88,10 @@ Note that at this point only arguments that are raw JSON types are supported:
 
 If you need marshalable Ruby types there instead, you might need to implement a custom `Serializer.`
 
+### Sqewer::SimpleJob
+
+The module `Sqewer::SimpleJob` can be included to a job class to add some features, specially dealing with attributes, see more details [here](https://github.com/WeTransfer/sqewer/blob/master/lib/sqewer/simple_job.rb).
+
 ## Jobs spawning dependent jobs
 
 If your `run` method on the job object accepts arguments (has non-zero `arity` ) the `ExecutionContext` will
@@ -119,11 +123,11 @@ include all the keyword arguments needed to instantiate the job when executing.
       def initialize(to:, body:)
         ...
       end
-      
+
       def run()
         ...
       end
-      
+
       def to_h
         {to: @to, body: @body}
       end
@@ -151,7 +155,7 @@ conform to the job serialization format used internally. For example, you can ha
         message = JSON.load(message_blob)
         return if message['Service'] # AWS test
         return HandleS3Notification.new(message) if message['Records']
-        
+
         super # as default
       end
     end
@@ -176,7 +180,7 @@ The very minimal executable for running jobs would be this:
 
     #!/usr/bin/env ruby
     require 'my_applicaion'
-    Sqewer::CLI.run
+    Sqewer::CLI.start
 
 This will connect to the queue at the URL set in the `SQS_QUEUE_URL` environment variable, and
 use all the default parameters. The `CLI` module will also set up a signal handler to terminate
@@ -233,9 +237,9 @@ you generate. For example, you could use a pipe. But in a more general case some
           ActiveRAMGobbler.fetch_stupendously_many_things.each do |...|
           end
         end
-        
+
         _, status = Process.wait2(pid)
-        
+
         # Raise an error in the parent process to signal Sqewer that the job failed
         # if the child exited with a non-0 status
         raise "Child process crashed" unless status.exitstatus && status.exitstatus.zero?
@@ -252,7 +256,7 @@ You can wrap job processing in middleware. A full-featured middleware class look
         # msg_id is the receipt handle, msg_payload is the message body string, msg_attributes are the message's attributes
         yield
       end
-      
+
       # Surrounds the actual job execution
       def around_execution(job, context)
         # job is the actual job you will be running, context is the ExecutionContext.
@@ -284,7 +288,7 @@ and traceable (make good use of logging).
 
 # Usage with Rails via ActiveJob
 
-This gem includes a queue adapter for usage with ActiveJob in Rails 4.2+. The functionality
+This gem includes a queue adapter for usage with ActiveJob in Rails 5+. The functionality
 is well-tested and should function for any well-conforming ActiveJob subclasses.
 
 To run the default `sqewer` worker setup against your Rails application, first set it as the
@@ -378,7 +382,7 @@ products using this library, was a very bad idea (more workload for deployment).
 
 ## Why so many configurable components?
 
-Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your 
+Because sometimes your requirements differ just-a-little-bit from what is provided, and you have to swap your
 implementation in instead. One product needs foreign-submitted SQS jobs (S3 notifications). Another product
 needs a custom Logger subclass. Yet another product needs process-based concurrency on top of threads.
 Yet another process needs to manage database connections when running the jobs. Have 3-4 of those, and a
@@ -398,7 +402,7 @@ Because I found that a producer-consumer model with a thread pool works quite we
 the Ruby standard library alone.
 
 ## Contributing to the library
- 
+
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
 * Fork the project.

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sqewer"
+
+require "irb"
+IRB.start(__FILE__)

data/lib/sqewer.rb

@@ -11,6 +11,11 @@ module Sqewer
     end
   end
 
+  # Returns a singleton of Aws::SQS::Client
+  def self.client
+    Sqewer::Connection.client
+  end
+
   # Loads a particular Sqewer extension that is not loaded
   # automatically during the gem require.
   #

data/lib/sqewer/cli.rb

@@ -1,3 +1,4 @@
+$stdout.sync = true
 # Wraps a Worker object in a process-wide commanline handler. Once the `start` method is
 # called, signal handlers will be installed for the following signals:
 #

data/lib/sqewer/connection.rb

@@ -39,12 +39,25 @@ class Sqewer::Connection
     raise "SQS_QUEUE_URL not set in the environment. This is the queue URL Sqewer uses by default."
   end
 
+  # Returns a singleton of Aws::SQS::Client
+  def self.client
+    # It's better using a singleton client to prevent making a lot of HTTP
+    # requests to the AWS metadata endpoint when getting credentials.
+    @client ||= begin
+      require 'aws-sdk-sqs'
+      ::Aws::SQS::Client.new(
+        instance_profile_credentials_timeout: 1,
+        instance_profile_credentials_retries: 5,
+      )
+    end
+  end
+
   # Initializes a new adapter, with access to the SQS queue at the given URL.
   #
   # @param queue_url[String] the SQS queue URL (the URL can be copied from your AWS console)
-  def initialize(queue_url)
-    require 'aws-sdk-sqs'
+  def initialize(queue_url, client: self.class.client)
     @queue_url = queue_url
+    @client = client
   end
 
   # Receive at most 10 messages from the queue, and return the array of Message objects. Retries for at
@@ -52,11 +65,15 @@ class Sqewer::Connection
   # even after 15 minutes it is either down or the server is misconfigured. Either way it makes no sense to
   # continue.
   #
-  # @return [Array<Message>] an array of Message objects 
+  # @return [Array<Message>] an array of Message objects
   def receive_messages
-    Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
-      response = client.receive_message(queue_url: @queue_url,
-        wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: BATCH_RECEIVE_SIZE)
+    Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_RECEIVE_FAILURES do
+      response = @client.receive_message(
+        queue_url: @queue_url,
+        attribute_names: ['All'],
+        wait_time_seconds: DEFAULT_TIMEOUT_SECONDS,
+        max_number_of_messages: BATCH_RECEIVE_SIZE
+      )
       response.messages.map {|message| Message.new(message.receipt_handle, message.body, message.attributes) }
     end
   end
@@ -65,7 +82,7 @@ class Sqewer::Connection
   #
   # @param message_body[String] the message to send
   # @param kwargs_for_send[Hash] additional arguments for the submit (such as `delay_seconds`).
-  # Passes the arguments to the AWS SDK. 
+  # Passes the arguments to the AWS SDK.
   # @return [void]
   def send_message(message_body, **kwargs_for_send)
     send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
@@ -187,9 +204,13 @@ class Sqewer::Connection
 
   private
 
+  def network_and_aws_sdk_errors
+    [NotOurFaultAwsError, Seahorse::Client::NetworkingError, Aws::SQS::Errors::InternalError]
+  end
+
   def handle_batch_with_retries(method, batch)
-    Retriable.retriable on: [NotOurFaultAwsError, Seahorse::Client::NetworkingError], tries: MAX_RANDOM_FAILURES_PER_CALL do
-      resp = client.send(method, queue_url: @queue_url, entries: batch)
+    Retriable.retriable on: network_and_aws_sdk_errors, tries: MAX_RANDOM_FAILURES_PER_CALL do
+      resp = @client.send(method, queue_url: @queue_url, entries: batch)
       wrong_messages, aws_failures = resp.failed.partition {|m| m.sender_fault }
       if wrong_messages.any?
         err = wrong_messages.inspect + ', ' + resp.inspect
@@ -201,11 +222,4 @@ class Sqewer::Connection
       end
     end
   end
-
-  def client
-    @client ||= Aws::SQS::Client.new(
-      instance_profile_credentials_timeout: 1, # defaults to 1 second
-      instance_profile_credentials_retries: 5, # defaults to 0 retries
-    )
-  end
 end

data/lib/sqewer/simple_job.rb

@@ -4,6 +4,13 @@
 # * initialize() will have keyword access to all accessors, and will ensure you have called each one of them
 # * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
 # * inspect() will provide a sensible default string representation for logging
+#
+# This module validates if the attributes defined in the job class are the same as
+# those persisted in the queue. More details on `Sqewer::SimpleJob#initialize`.
+# Because of this, it's required to create a new job class when adding or removing
+# an attribute.
+# This mechanism guarantees strong consistency. Without it, a new deployed job class
+# could process old incompatible payloads.
 module Sqewer::SimpleJob
   UnknownJobAttribute = Class.new(Sqewer::Error)
   MissingAttribute = Class.new(Sqewer::Error)
@@ -53,7 +60,7 @@ module Sqewer::SimpleJob
       accessor = "#{k}="
       touched_attributes << k
       unless respond_to?(accessor)
-        raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}" 
+        raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}"
       end
 
       send("#{k}=", v)

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '6.4.0'
+  VERSION = '8.0.0'
 end

data/lib/sqewer/worker.rb

@@ -91,7 +91,7 @@ class Sqewer::Worker
 
     consumers = (1..@num_threads).map do
       Thread.new do
-        catch(:goodbye) { loop {take_and_execute} }
+        loop { take_and_execute }
       end
     end
 
@@ -233,8 +233,11 @@ class Sqewer::Worker
     message = @execution_queue.pop(nonblock=true)
     handle_message(message)
   rescue ThreadError # Queue is empty
-    throw :goodbye if stopping?
-    sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+    if stopping?
+      Thread.current.exit
+    else
+      sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+    end
   rescue => e # anything else, at or below StandardError that does not need us to quit
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..64], e.class, e.message] }
     e.backtrace.each { |s| @logger.debug{"\t#{s}"} }

data/sqewer.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Process jobs from SQS}
   spec.description   = %q{A full-featured library for all them SQS worker needs}
   spec.homepage      = "https://github.com/WeTransfer/sqewer"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the "allowed_push_host"
   # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -36,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "retriable"
 
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 
   # The Rails deps can be relaxed, they are specified more exactly in the gemfiles/

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 6.4.0
+  version: 8.0.0
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2021-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -101,14 +101,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -226,6 +226,7 @@ email:
 - me@julik.nl
 - linkyndy@gmail.com
 executables:
+- console
 - sqewer
 - sqewer_rails
 extensions: []
@@ -239,10 +240,10 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- bin/console
 - bin/sqewer
 - bin/sqewer_rails
 - example.env
-- gemfiles/Gemfile.rails-4.2.x
 - gemfiles/Gemfile.rails-5.0.x
 - gemfiles/Gemfile.rails-5.1.x
 - lib/sqewer.rb
@@ -278,14 +279,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Process jobs from SQS

data/gemfiles/Gemfile.rails-4.2.x

@@ -1,8 +0,0 @@
-source "http://rubygems.org"
-
-# Gemspec as base dependency set
-gemspec path: __dir__ + '/..'
-
-gem 'sqlite3', '~> 1.3.6'
-gem 'activejob', "~> 4.2.0"
-gem 'activerecord', "~> 4.2.0"