sqewer 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d92c1ce7fa4d3baa8d3d24c20f5f806be7e3d037
-  data.tar.gz: 26a18a95cbb014ad94b77fbabee3af60a4b6eec8
+  metadata.gz: d580e81c48aeb4f31bf0069c603c3cee0413c51d
+  data.tar.gz: 87d546de3454ff297750d70d355a71c45340cc0a
 SHA512:
-  metadata.gz: 22ebdc336e6c9ba4d6f0c4d7d39fcc90961e8400c42f9988a876e302ddf32d6b51b2db4494da174d9d20beef1a05cfe4a5d7b0a5657a067c1d3a1b9534086922
-  data.tar.gz: 3e7c5d7cca071767ab7629cedfcc76e8ba1c07452324ebc0bad1095a2e52db11e2345859e760658797c3f892d2600af984fc280e3a9c2c99edd5ee931ef084e4
+  metadata.gz: 4be05bef37befcb1e3f49021be38af632bbb4af52bfac17a4bc66fa9f81bca65406416538a59e6c94faac584f6d60884e43a46547cc5c96a9e900cd8eef7532e
+  data.tar.gz: 9bcf12c6ff462a7ca49659077193c1b95568872c4dd38f8444899000434cf8ce83d79e555bfd7e6226c4b18f9b2fefb72cc3feec28687b65c3047fe0b7858965

data/DETAILS.md

@@ -159,7 +159,7 @@ arguments:
 
     class CustomWorker < Sqewer::Worker
       def initialize(**kwargs)
-        super(serializer: CustomSerializer.mnew, ..., **kwargs)
+        super(serializer: CustomSerializer.new, ..., **kwargs)
       end
     end
 

data/lib/sqewer/connection.rb

@@ -40,7 +40,6 @@ class Sqewer::Connection
   #
   # @return [Array<Message>] an array of Message objects 
   def receive_messages
-    client = ::Aws::SQS::Client.new
     response = client.receive_message(queue_url: @queue_url,
       wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: 10)
     response.messages.map do | message |
@@ -58,6 +57,7 @@ class Sqewer::Connection
     send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
   end
   
+  # Stores the messages for the SQS queue (both deletes and sends), and yields them in allowed batch sizes
   class MessageBuffer < Struct.new(:messages)
     MAX_RECORDS = 10
     def initialize
@@ -68,6 +68,7 @@ class Sqewer::Connection
     end
   end
   
+  # Saves the messages to send to the SQS queue
   class SendBuffer < MessageBuffer
     def send_message(message_body, **kwargs_for_send)
       # The "id" is only valid _within_ the request, and is used when
@@ -78,6 +79,7 @@ class Sqewer::Connection
     end
   end
   
+  # Saves the receipt handles to batch-delete from the SQS queue
   class DeleteBuffer < MessageBuffer
     def delete_message(receipt_handle)
       # The "id" is only valid _within_ the request, and is used when
@@ -94,7 +96,6 @@ class Sqewer::Connection
   def send_multiple_messages
     buffer = SendBuffer.new
     yield(buffer)
-    client = ::Aws::SQS::Client.new
     buffer.each_batch do | batch |
       resp = client.send_message_batch(queue_url: @queue_url, entries: batch)
       failed = resp.failed
@@ -121,7 +122,6 @@ class Sqewer::Connection
     buffer = DeleteBuffer.new
     yield(buffer)
     
-    client = ::Aws::SQS::Client.new
     buffer.each_batch do | batch |
       resp = client.delete_message_batch(queue_url: @queue_url, entries: batch)
       failed = resp.failed
@@ -131,4 +131,30 @@ class Sqewer::Connection
       end
     end
   end
+  
+  private
+  
+  class RetryWrapper < Struct.new(:sqs_client)
+    MAX_RETRIES = 1000
+    # Provide retrying wrappers for all the methods of Aws::SQS::Client that we actually use
+    [:delete_message_batch, :send_message_batch, :receive_message].each do |retriable_method_name|
+      define_method(retriable_method_name) do |*args, **kwargs|
+        tries = 1
+        begin
+          sqs_client.public_send(retriable_method_name, *args, **kwargs)
+        rescue Seahorse::Client::NetworkingError => e
+          if (tries += 1) >= MAX_RETRIES
+            raise(e)
+          else
+            sleep 0.5
+            retry
+          end
+        end
+      end
+    end
+  end
+  
+  def client
+    @client ||= RetryWrapper.new(Aws::SQS::Client.new)
+  end
 end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '4.1.0'
+  VERSION = '4.2.0'
 end

data/lib/sqewer/worker.rb

@@ -189,7 +189,6 @@ class Sqewer::Worker
   
   def handle_message(message)
     return unless message.receipt_handle
-    Thread.current[:queue_messsage] = '%s...' %message.body[0..32]
     return @connection.delete_message(message.receipt_handle) unless message.has_body?
     @isolator.perform(self, message)
     # The message delete happens within the Isolator

data/spec/sqewer/connection_spec.rb

@@ -29,6 +29,8 @@ describe Sqewer::Connection do
       expect(conn).to receive(:send_multiple_messages).and_call_original
       conn.send_message('abcdef', delay_seconds: 5)
     end
+    
+    it 'retries on networking errors'
   end
   
   describe '#send_multiple_messages' do
@@ -69,6 +71,9 @@ describe Sqewer::Connection do
         end
       }.to raise_error(/messages failed to send/)
     end
+    
+    it 'retries on networking errors'
+    
   end
   
   describe '#delete_message' do
@@ -112,6 +117,8 @@ describe Sqewer::Connection do
         end
       }.to raise_error(/messages failed to delete/)
     end
+    
+    it 'retries on networking errors'
   end
   
   describe '#receive_messages' do
@@ -132,5 +139,7 @@ describe Sqewer::Connection do
       messages = s.receive_messages
       expect(messages.length).to eq(5)
     end
+    
+    it 'retries on networking errors'
   end
 end

data/sqewer.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sqewer 4.1.0 ruby lib
+# stub: sqewer 4.2.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sqewer"
-  s.version = "4.1.0"
+  s.version = "4.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-02-07"
+  s.date = "2016-02-12"
   s.description = "Process jobs from SQS"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -44,7 +44,6 @@ Gem::Specification.new do |s|
     "lib/sqewer/submitter.rb",
     "lib/sqewer/version.rb",
     "lib/sqewer/worker.rb",
-    "spec/conveyor_belt_spec.rb",
     "spec/spec_helper.rb",
     "spec/sqewer/atomic_counter_spec.rb",
     "spec/sqewer/cli_app.rb",
@@ -56,6 +55,7 @@ Gem::Specification.new do |s|
     "spec/sqewer/simple_job_spec.rb",
     "spec/sqewer/submitter_spec.rb",
     "spec/sqewer/worker_spec.rb",
+    "spec/sqewer_module_spec.rb",
     "sqewer.gemspec"
   ]
   s.homepage = "https://gitlab.wetransfer.net/julik/sqewer"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-07 00:00:00.000000000 Z
+date: 2016-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -225,7 +225,6 @@ files:
 - lib/sqewer/submitter.rb
 - lib/sqewer/version.rb
 - lib/sqewer/worker.rb
-- spec/conveyor_belt_spec.rb
 - spec/spec_helper.rb
 - spec/sqewer/atomic_counter_spec.rb
 - spec/sqewer/cli_app.rb
@@ -237,6 +236,7 @@ files:
 - spec/sqewer/simple_job_spec.rb
 - spec/sqewer/submitter_spec.rb
 - spec/sqewer/worker_spec.rb
+- spec/sqewer_module_spec.rb
 - sqewer.gemspec
 homepage: https://gitlab.wetransfer.net/julik/sqewer
 licenses: