sqewer 3.0.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edded56859437d9b7bce9424d2c00b4a8d999756
-  data.tar.gz: aaa87e0f5ab8a4fe1b0c73682e4282bd7b52534b
+  metadata.gz: 3df44f7b0b7b55d48c513d4573c16c22051a6452
+  data.tar.gz: 2967480bcb890b11fa117135e7a4f3541d9724be
 SHA512:
-  metadata.gz: 8c0394391564b80cd5148dfb4446561ae29902e4e05fc62360b0d0fa454effb25d6bc62978f3b2a81fcd3021c7cc59afbad8031c6bb5fcef1cfd18965e87f233
-  data.tar.gz: 7daa39ca0ee05abf18c65f9bfbe59d334160522243a3a1fb0589ba413070893d89f0896e57597506678459951337bb29a4fa686fb6442d99ae47aef8f7f65d35
+  metadata.gz: 8fd9e0949bf88525d352b70b46899e91ecef4865d2b60a64348d5ea3587052e18fa6d3d05d12f62a46f31420eac0a80920ebbe4273242d1f2f85a834e9e249f5
+  data.tar.gz: 2f06335925254b2e923fce2bb979dc10cfa10232a61f16d6ffd47d745678204889167504f008b45c3bdd38419595b7b51d135306ea0f0782548c9472f122e6d0

data/lib/sqewer/connection_messagebox.rb

@@ -0,0 +1,50 @@
+require 'thread'
+
+# A recorder for send_message and delete_message calls.
+# Will buffer those calls as if it were a Connection, and then execute
+# them within a synchronized mutex lock, to prevent concurrent submits
+# from the Connection object
+class Sqewer::ConnectionMessagebox
+  class MethodCall < Struct.new(:method_name, :posargs, :kwargs)
+    def perform(on)
+      if kwargs && posargs
+        on.public_send(method_name, *posargs, **kwargs)
+      elsif kwargs
+        on.public_send(method_name, **kwargs)
+      elsif posargs
+        on.public_send(method_name, *posargs)
+      else
+        on.public_send(method_name)
+      end
+    end
+  end
+  
+  def initialize(connection)
+    @connection = connection
+    @queue = Queue.new
+    @mux = Mutex.new
+  end
+  
+  def receive_messages
+    @connection.receive_messages
+  end
+  
+  def send_message(message_body, **kwargs_for_send)
+    @queue << MethodCall.new(:send_message, [message_body], kwargs_for_send)
+  end
+  
+  def delete_message(message_identifier)
+    @queue << MethodCall.new(:delete_message, [message_identifier], nil)
+  end
+  
+  def flush!
+    @mux.synchronize do
+      executed = 0
+      while @queue.length.nonzero?
+        @queue.pop.perform(@connection)
+        executed += 1
+      end
+      executed
+    end
+  end
+end

data/lib/sqewer/isolator.rb

@@ -39,23 +39,31 @@ class Sqewer::Isolator
         worker.submitter_class, worker.execution_context_class,
           worker.middleware_stack, worker.connection, worker.serializer, worker.logger
     
+    box = Sqewer::ConnectionMessagebox.new(connection)
+    
     job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
       serializer.unserialize(message.body)
     end
     return unless job
     
-    submitter = submitter_class.new(connection, serializer)
+    submitter = submitter_class.new(box, serializer)
     context = execution_context_class.new(submitter, {'logger' => logger})
     
     t = Time.now
     middleware_stack.around_execution(job, context) do
       job.method(:run).arity.zero? ? job.run : job.run(context)
     end
-    # FIXME: do not mix templating and format strings
+    
+    # Perform two flushes, one for any possible jobs the job has spawned,
+    # and one for the job delete afterwards
+    box.delete_message(message.receipt_handle)
+    
     delta = Time.now - t
     logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
   rescue => e
-    logger.error { "[worker] Failed #{job.inspect} with a #{e}" } if job
     raise e
+  ensure
+    n_flushed = box.flush!
+    logger.info { "[worker] Flushed %d messages" % n_flushed } if n_flushed.nonzero?
   end
 end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '3.0.1'
+  VERSION = '4.0.0'
 end

data/lib/sqewer/worker.rb

@@ -95,8 +95,8 @@ class Sqewer::Worker
     consumers = (1..@num_threads).map do
       Thread.new do
         loop { 
-          break if @state.in_state?(:stopping)
           take_and_execute
+          break if stopping?
         }
       end
     end
@@ -180,7 +180,7 @@ class Sqewer::Worker
     return unless message.receipt_handle
     return @connection.delete_message(message.receipt_handle) unless message.has_body?
     @isolator.perform(self, message)
-    @connection.delete_message(message.receipt_handle)
+    # The message delete happens within the Isolator
   end
   
   def take_and_execute
@@ -190,11 +190,7 @@ class Sqewer::Worker
     sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
     Thread.pass
   rescue => e # anything else, at or below StandardError that does not need us to quit
-    @logger.error { "[worker] Failed #{message.inspect} with #{e}" }
-    @logger.error(e.class)
-    @logger.error(e.message)
+    @logger.error { "[worker] Failed %s... with %s: %s" % [message.inspect[0..64], e.class, e.message] }
     e.backtrace.each { |s| @logger.error{"\t#{s}"} }
   end
-  
-  STR_logger = 'logger'
 end

data/sqewer.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sqewer 3.0.1 ruby lib
+# stub: sqewer 4.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sqewer"
-  s.version = "3.0.1"
+  s.version = "4.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-02-03"
+  s.date = "2016-02-04"
   s.description = "Process jobs from SQS"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
     "lib/sqewer/atomic_counter.rb",
     "lib/sqewer/cli.rb",
     "lib/sqewer/connection.rb",
+    "lib/sqewer/connection_messagebox.rb",
     "lib/sqewer/contrib/appsignal_wrapper.rb",
     "lib/sqewer/contrib/performable.rb",
     "lib/sqewer/execution_context.rb",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-03 00:00:00.000000000 Z
+date: 2016-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -212,6 +212,7 @@ files:
 - lib/sqewer/atomic_counter.rb
 - lib/sqewer/cli.rb
 - lib/sqewer/connection.rb
+- lib/sqewer/connection_messagebox.rb
 - lib/sqewer/contrib/appsignal_wrapper.rb
 - lib/sqewer/contrib/performable.rb
 - lib/sqewer/execution_context.rb