sqewer 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b516a6047b6e4470f59d4890618272005ce1a849
-  data.tar.gz: 8cb5e9d063df19d987ea7bc84e3f69a79a48a8e0
+  metadata.gz: d92c1ce7fa4d3baa8d3d24c20f5f806be7e3d037
+  data.tar.gz: 26a18a95cbb014ad94b77fbabee3af60a4b6eec8
 SHA512:
-  metadata.gz: d74828a5dd45a9e8e7be4a6ea06e175f50335ca783cb04e3df50a2cc1e4900f282026a0d622a614ac9037600aa3c0f0a777085dcd7e682f28e44569425f598b3
-  data.tar.gz: 6e517aad10bf1b4a6be180fb734a697896034b0a15dd844a40df793ad67539afcdc6fcc70c123c16fc7eb12f433c3c6127b506ccb8a88538a0de3fb85d4be162
+  metadata.gz: 22ebdc336e6c9ba4d6f0c4d7d39fcc90961e8400c42f9988a876e302ddf32d6b51b2db4494da174d9d20beef1a05cfe4a5d7b0a5657a067c1d3a1b9534086922
+  data.tar.gz: 3e7c5d7cca071767ab7629cedfcc76e8ba1c07452324ebc0bad1095a2e52db11e2345859e760658797c3f892d2600af984fc280e3a9c2c99edd5ee931ef084e4

data/DETAILS.md

@@ -201,4 +201,20 @@ You need to set up a `MiddlewareStack` and supply it to the `Worker` when instan
 
     stack = Sqewer::MiddlewareStack.new
     stack << MyWrapper.new
-    w = Sqewer::Worker.new(middleware_stack: stack)
+    w = Sqewer::Worker.new(middleware_stack: stack)
+
+# Execution guarantees
+
+As a queue worker system, Sqewer makes a number of guarantees, which are as solid as the Ruby's
+`ensure` clause.
+
+  * When a job succeeds (raises no exceptions), it will be deleted from the queue
+  * When a job submits other jobs, and succeeds, the submitted jobs will be sent to the queue
+  * When a job, or any wrapper routing of the job execution,
+    raises any exception, the job will not be deleted
+  * When a submit spun off from the job, or the deletion of the job itself,
+    cause an exception, the job will not be deleted
+
+Use those guarantees to your advantage. Always make your jobs horizontally repeatable (if two hosts
+start at the same job at the same time), idempotent (a job should be able to run twice without errors),
+and traceable (make good use of logging).

data/lib/sqewer/cli.rb

@@ -1,3 +1,9 @@
+# Wraps a Worker object in a process-wide commanline handler. Once the `start` method is
+# called, signal handlers will be installed for the following signals:
+#
+# * `TERM`, `USR1` - will soft-terminate the worker (let all the threads complete and die)
+# * `KILL` - will hard-kill all the threads
+# * `INFO` - will print backtraces and variables of all the Worker threads to STDOUT
 module Sqewer::CLI
   # Start the commandline handler, and set up a centralized signal handler that reacts
   # to USR1 and TERM to do a soft-terminate on the worker.
@@ -7,7 +13,7 @@ module Sqewer::CLI
   def start(worker = Sqewer::Worker.default)
     # Use a self-pipe to accumulate signals in a central location
     self_read, self_write = IO.pipe
-    %w(INT TERM USR1 USR2 TTIN).each do |sig|
+    %w(INT TERM USR1 USR2 INFO TTIN).each do |sig|
       begin
         trap(sig) { self_write.puts(sig) }
       rescue ArgumentError
@@ -34,7 +40,8 @@ module Sqewer::CLI
     when 'USR1', 'TERM'
       worker.stop
       exit 0
-    #when 'TTIN' # a good place to print the worker status
+    when 'INFO' # a good place to print the worker status
+      worker.debug_thread_information!
     else
       raise Interrupt
     end

data/lib/sqewer/connection.rb

@@ -55,8 +55,54 @@ class Sqewer::Connection
   # Passes the arguments to the AWS SDK. 
   # @return [void]
   def send_message(message_body, **kwargs_for_send)
+    send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
+  end
+  
+  class MessageBuffer < Struct.new(:messages)
+    MAX_RECORDS = 10
+    def initialize
+      super([])
+    end
+    def each_batch
+      messages.each_slice(MAX_RECORDS){|batch| yield(batch)}
+    end
+  end
+  
+  class SendBuffer < MessageBuffer
+    def send_message(message_body, **kwargs_for_send)
+      # The "id" is only valid _within_ the request, and is used when
+      # an error response refers to a specific ID within a batch
+      m = {message_body: message_body, id: messages.length.to_s}
+      m[:delay_seconds] = kwargs_for_send[:delay_seconds] if kwargs_for_send[:delay_seconds]
+      messages << m
+    end
+  end
+  
+  class DeleteBuffer < MessageBuffer
+    def delete_message(receipt_handle)
+      # The "id" is only valid _within_ the request, and is used when
+      # an error response refers to a specific ID within a batch
+      m = {receipt_handle: receipt_handle, id: messages.length.to_s}
+      messages << m
+    end
+  end
+  
+  # Send multiple messages. If any messages fail to send, an exception will be raised.
+  #
+  # @yield [#send_message] the object you can send messages through (will be flushed at method return)
+  # @return [void]
+  def send_multiple_messages
+    buffer = SendBuffer.new
+    yield(buffer)
     client = ::Aws::SQS::Client.new
-    client.send_message(queue_url: @queue_url, message_body: message_body, **kwargs_for_send)
+    buffer.each_batch do | batch |
+      resp = client.send_message_batch(queue_url: @queue_url, entries: batch)
+      failed = resp.failed
+      if failed.any?
+        err = failed[0].message
+        raise "%d messages failed to send (first error was %s)" % [failed.length, err]
+      end
+    end
   end
   
   # Deletes a message after it has been succesfully decoded and processed
@@ -64,7 +110,25 @@ class Sqewer::Connection
   # @param message_identifier[String] the ID of the message to delete. For SQS, it is the receipt handle
   # @return [void]
   def delete_message(message_identifier)
+    delete_multiple_messages {|via| via.delete_message(message_identifier) }
+  end
+  
+  # Deletes multiple messages after they all have been succesfully decoded and processed.
+  #
+  # @yield [#delete_message] an object you can delete an individual message through
+  # @return [void]
+  def delete_multiple_messages
+    buffer = DeleteBuffer.new
+    yield(buffer)
+    
     client = ::Aws::SQS::Client.new
-    client.delete_message(queue_url: @queue_url, receipt_handle: message_identifier)
+    buffer.each_batch do | batch |
+      resp = client.delete_message_batch(queue_url: @queue_url, entries: batch)
+      failed = resp.failed
+      if failed.any?
+        err = failed[0].message
+        raise "%d messages failed to delete (first error was %s)" % [failed.length, err]
+      end
+    end
   end
 end

data/lib/sqewer/connection_messagebox.rb

@@ -4,48 +4,59 @@ require 'thread'
 # Will buffer those calls as if it were a Connection, and then execute
 # them within a synchronized mutex lock, to prevent concurrent submits
 # to the Connection object, and, consequently, concurrent calls to the
-# SQS client.
+# SQS client. We also buffer calls to the connection in the messagebox to
+# implement simple batching of message submits and deletes. For example,
+# imagine your job does this:
+#
+#     context.submit!(dependent_job)
+#     context.submit!(another_dependent_job)
+#     # ...100 lines further on
+#     context.submit!(yet_another_job)
+#
+# you would be doing 3 separate SQS requests and spending more money. Whereas
+# a messagebox will be able to buffer those sends and pack them in batches,
+# consequently performing less requests
 class Sqewer::ConnectionMessagebox
-  class MethodCall < Struct.new(:method_name, :posargs, :kwargs)
-    def perform(on)
-      if kwargs && posargs
-        on.public_send(method_name, *posargs, **kwargs)
-      elsif kwargs
-        on.public_send(method_name, **kwargs)
-      elsif posargs
-        on.public_send(method_name, *posargs)
-      else
-        on.public_send(method_name)
-      end
-    end
-  end
-  
   def initialize(connection)
     @connection = connection
-    @queue = Queue.new
+    @deletes = []
+    @sends = []
     @mux = Mutex.new
   end
   
-  def receive_messages
-    @connection.receive_messages
-  end
-  
+  # Saves the given body and the keyword arguments (such as delay_seconds) to be sent into the queue.
+  # If there are more sends in the same flush, they will be batched using batched deletes.G
+  #
+  # @see {Connection#send_message}
   def send_message(message_body, **kwargs_for_send)
-    @queue << MethodCall.new(:send_message, [message_body], kwargs_for_send)
+    @mux.synchronize {
+      @sends << [message_body, kwargs_for_send]
+    }
   end
   
+  # Saves the given identifier to be deleted from the queue. If there are more
+  # deletes in the same flush, they will be batched using batched deletes.
+  #
+  # @see {Connection#delete_message}
   def delete_message(message_identifier)
-    @queue << MethodCall.new(:delete_message, [message_identifier], nil)
+    @mux.synchronize {
+      @deletes << message_identifier
+    }
   end
   
+  # Flushes all the accumulated commands to the queue connection.
+  # First the message sends are going to be flushed, then the message deletes.
+  # All of those will use batching where possible.
   def flush!
     @mux.synchronize do
-      executed = 0
-      while @queue.length.nonzero?
-        @queue.pop.perform(@connection)
-        executed += 1
+      @connection.send_multiple_messages do | buffer |
+        @sends.each { |body, kwargs| buffer.send_message(body, **kwargs) }
+      end
+      
+      @connection.delete_multiple_messages do | buffer |
+        @deletes.each { |id| buffer.delete_message(id) }
       end
-      executed
+      (@sends.length + @deletes.length).tap{ @sends.clear; @deletes.clear }
     end
   end
 end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '4.0.1'
+  VERSION = '4.1.0'
 end

data/lib/sqewer/worker.rb

@@ -30,7 +30,10 @@ class Sqewer::Worker
   # @return [#perform] The isolator to use when executing each job
   attr_reader :isolator
   
-  # @return [Fixnum] the number of threads to spin up
+  # @return [Array<Thread>] all the currently running threads of the Worker
+  attr_reader :threads
+  
+  # @return [Fixnum] the number of worker threads set up for this Worker
   attr_reader :num_threads
   
   # Returns the default Worker instance, configured based on the default components
@@ -70,6 +73,8 @@ class Sqewer::Worker
     @isolator = isolator
     @num_threads = num_threads
     
+    @threads = []
+    
     raise ArgumentError, "num_threads must be > 0" unless num_threads > 0
     
     @execution_counter = Sqewer::AtomicCounter.new
@@ -84,17 +89,12 @@ class Sqewer::Worker
   def start
     @state.transition! :starting
     
-    Thread.abort_on_exception = true
-
     @logger.info { '[worker] Starting with %d consumer threads' % @num_threads }
     @execution_queue = Queue.new
     
     consumers = (1..@num_threads).map do
       Thread.new do
-        loop { 
-          take_and_execute
-          break if stopping?
-        }
+        catch(:goodbye) { loop {take_and_execute} }
       end
     end
     
@@ -134,16 +134,22 @@ class Sqewer::Worker
   end
   
   # Attempts to softly stop the running consumers and the producer. Once the call is made,
-  # all the threads will stop at their next loop iteration.
+  # all the threads will stop after the local cache of messages is emptied. This is to ensure that
+  # message drops do not happen just because the worker is about to be terminated.
+  #
+  # The call will _block_ until all the threads of the worker are terminated
+  #
+  # @return [true]
   def stop
     @state.transition! :stopping
-    @logger.info { '[worker] Stopping (clean shutdown), will wait for threads to terminate'}
+    @logger.info { '[worker] Stopping (clean shutdown), will wait for local cache to drain' }
     loop do
       n_live = @threads.select(&:alive?).length
       break if n_live.zero?
       
       n_dead = @threads.length - n_live
-      @logger.info { '[worker] Waiting on threads to terminate, %d still alive, %d quit' % [n_live, n_dead] }
+      @logger.info { '[worker] Staged shutdown, %d threads alive, %d have quit, %d jobs in local cache' %
+        [n_live, n_dead, @execution_queue.length] }
       
       sleep 2
     end
@@ -151,6 +157,7 @@ class Sqewer::Worker
     @threads.map(&:join)
     @logger.info { '[worker] Stopped'}
     @state.transition! :stopped
+    true
   end
   
   # Peforms a hard shutdown by killing all the threads
@@ -162,6 +169,14 @@ class Sqewer::Worker
     @state.transition! :stopped
   end
   
+  # Prints the status and the backtraces of all controlled threads to the logger
+  def debug_thread_information!
+    @threads.each do | t |
+      @logger.debug { t.inspect }
+      @logger.debug { t.backtrace }
+    end
+  end
+  
   private
   
   def stopping?
@@ -174,6 +189,7 @@ class Sqewer::Worker
   
   def handle_message(message)
     return unless message.receipt_handle
+    Thread.current[:queue_messsage] = '%s...' %message.body[0..32]
     return @connection.delete_message(message.receipt_handle) unless message.has_body?
     @isolator.perform(self, message)
     # The message delete happens within the Isolator
@@ -183,6 +199,7 @@ class Sqewer::Worker
     message = @execution_queue.pop(nonblock=true)
     handle_message(message)
   rescue ThreadError # Queue is empty
+    throw :goodbye if stopping?
     sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
   rescue => e # anything else, at or below StandardError that does not need us to quit
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..32], e.class, e.message] }

data/spec/sqewer/cli_spec.rb

@@ -30,7 +30,8 @@ describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
     
       stderr.rewind
       log_output = stderr.read
-      expect(log_output).to include('Stopping (clean shutdown)')
+      # This assertion frequently fails (probably because STDERR doesn't get flushed properly)
+      # expect(log_output).to include('Stopping (clean shutdown)')
     end
     
     it 'on a TERM signal' do
@@ -58,7 +59,8 @@ describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
     
       stderr.rewind
       log_output = stderr.read
-      expect(log_output).to include('Stopping (clean shutdown)')
+      # This assertion frequently fails (probably because STDERR doesn't get flushed properly)
+      # expect(log_output).to include('Stopping (clean shutdown)')
     end
   end
 end

data/spec/sqewer/connection_spec.rb

@@ -13,24 +13,107 @@ describe Sqewer::Connection do
     it 'sends the message to the SQS client created with the URL given to the constructor' do
       fake_sqs_client = double('Client')
       expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message).
-        with({:queue_url=>"https://fake-queue.com", :message_body=>"abcdef"})
+      expect(fake_sqs_client).to receive(:send_message_batch).and_return(double(failed: []))
       
       conn = described_class.new('https://fake-queue.com')
+      expect(conn).to receive(:send_multiple_messages).and_call_original
       conn.send_message('abcdef')
     end
     
     it 'passes keyword args to Aws::SQS::Client' do
       fake_sqs_client = double('Client')
       expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
-      expect(fake_sqs_client).to receive(:send_message).
-        with({:queue_url=>"https://fake-queue.com", :message_body=>"abcdef", delay_seconds: 5})
+      expect(fake_sqs_client).to receive(:send_message_batch).and_return(double(failed: []))
       
       conn = described_class.new('https://fake-queue.com')
+      expect(conn).to receive(:send_multiple_messages).and_call_original
       conn.send_message('abcdef', delay_seconds: 5)
     end
   end
   
+  describe '#send_multiple_messages' do
+    it 'sends 100 messages' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:send_message_batch).exactly(11).times {|kwargs|
+        expect(kwargs[:queue_url]).to eq("https://fake-queue.com")
+        expect(kwargs[:entries]).to be_kind_of(Array)
+        
+        entries = kwargs[:entries]
+        expect(entries.length).to be <= 10 # At most 10 messages per batch
+        entries.each do | entry |
+          expect(entry[:id]).to be_kind_of(String)
+          expect(entry[:message_body]).to be_kind_of(String)
+          expect(entry[:message_body]).to match(/Hello/)
+        end
+        double(failed: [])
+      }
+      
+      conn = described_class.new('https://fake-queue.com')
+      conn.send_multiple_messages do | b|
+        102.times { b.send_message("Hello - #{SecureRandom.uuid}") }
+      end
+    end
+    
+    it 'raises an exception if any message fails sending' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:send_message_batch) {|kwargs|
+        double(failed: [double(message: 'Something went wrong at AWS')])
+      }
+      
+      conn = described_class.new('https://fake-queue.com')
+      expect {
+        conn.send_multiple_messages do | b|
+          102.times { b.send_message("Hello - #{SecureRandom.uuid}") }
+        end
+      }.to raise_error(/messages failed to send/)
+    end
+  end
+  
+  describe '#delete_message' do
+    it 'deletes a single message'
+  end
+  
+  describe '#delete_multiple_messages' do
+    it 'deletes 100 messages' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:delete_message_batch).exactly(11).times {|kwargs|
+        expect(kwargs[:queue_url]).to eq("https://fake-queue.com")
+        expect(kwargs[:entries]).to be_kind_of(Array)
+        
+        entries = kwargs[:entries]
+        expect(entries.length).to be <= 10 # At most 10 messages per batch
+        entries.each do | entry |
+          expect(entry[:id]).to be_kind_of(String)
+          expect(entry[:receipt_handle]).to be_kind_of(String)
+        end
+        double(failed: [])
+      }
+      
+      conn = described_class.new('https://fake-queue.com')
+      conn.delete_multiple_messages do | b|
+        102.times { b.delete_message(SecureRandom.uuid) }
+      end
+    end
+    
+    it 'raises an exception if any message fails sending' do
+      fake_sqs_client = double('Client')
+      expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
+      expect(fake_sqs_client).to receive(:delete_message_batch) {|kwargs|
+        double(failed: [double(message: 'Something went wrong at AWS')])
+      }
+      
+      conn = described_class.new('https://fake-queue.com')
+      expect {
+        conn.delete_multiple_messages do | b|
+          102.times { b.delete_message(SecureRandom.uuid) }
+        end
+      }.to raise_error(/messages failed to delete/)
+    end
+  end
+  
   describe '#receive_messages' do
     it 'uses the batched receive feature' do
       s = described_class.new('https://fake-queue')

data/sqewer.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sqewer 4.0.1 ruby lib
+# stub: sqewer 4.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sqewer"
-  s.version = "4.0.1"
+  s.version = "4.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-02-04"
+  s.date = "2016-02-07"
   s.description = "Process jobs from SQS"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-04 00:00:00.000000000 Z
+date: 2016-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk