sqewer 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3df44f7b0b7b55d48c513d4573c16c22051a6452
-  data.tar.gz: 2967480bcb890b11fa117135e7a4f3541d9724be
+  metadata.gz: b516a6047b6e4470f59d4890618272005ce1a849
+  data.tar.gz: 8cb5e9d063df19d987ea7bc84e3f69a79a48a8e0
 SHA512:
-  metadata.gz: 8fd9e0949bf88525d352b70b46899e91ecef4865d2b60a64348d5ea3587052e18fa6d3d05d12f62a46f31420eac0a80920ebbe4273242d1f2f85a834e9e249f5
-  data.tar.gz: 2f06335925254b2e923fce2bb979dc10cfa10232a61f16d6ffd47d745678204889167504f008b45c3bdd38419595b7b51d135306ea0f0782548c9472f122e6d0
+  metadata.gz: d74828a5dd45a9e8e7be4a6ea06e175f50335ca783cb04e3df50a2cc1e4900f282026a0d622a614ac9037600aa3c0f0a777085dcd7e682f28e44569425f598b3
+  data.tar.gz: 6e517aad10bf1b4a6be180fb734a697896034b0a15dd844a40df793ad67539afcdc6fcc70c123c16fc7eb12f433c3c6127b506ccb8a88538a0de3fb85d4be162

data/Gemfile

@@ -1,7 +1,7 @@
 source "http://rubygems.org"
 
 gem 'aws-sdk', '~> 2'
-gem 'very_tiny_state_machine', '~> 1'
+gem 'very_tiny_state_machine'
 gem 'hash_tools'
 gem 'exceptional_fork'
 

data/lib/sqewer/connection_messagebox.rb

@@ -3,7 +3,8 @@ require 'thread'
 # A recorder for send_message and delete_message calls.
 # Will buffer those calls as if it were a Connection, and then execute
 # them within a synchronized mutex lock, to prevent concurrent submits
-# from the Connection object
+# to the Connection object, and, consequently, concurrent calls to the
+# SQS client.
 class Sqewer::ConnectionMessagebox
   class MethodCall < Struct.new(:method_name, :posargs, :kwargs)
     def perform(on)

data/lib/sqewer/isolator.rb

@@ -33,12 +33,17 @@ class Sqewer::Isolator
   # @param worker[Sqewer::Worker] the worker that is running the jobs
   # @param message[Sqewer::Connection::Message] the message that is being processed
   def perform(worker, message)
+    submitter_class = worker.submitter_class
+    execution_context_class = worker.execution_context_class
+    middleware_stack = worker.middleware_stack
+    connection = worker.connection
+    logger = worker.logger
+    serializer = worker.serializer
     
-    submitter_class, execution_context_class, 
-      middleware_stack, connection, serializer, logger = 
-        worker.submitter_class, worker.execution_context_class,
-          worker.middleware_stack, worker.connection, worker.serializer, worker.logger
-    
+    # Create a messagebox that buffers all the calls to Connection, so that
+    # we can send out those commands in one go (without interfering with senders
+    # on other threads, as it seems the Aws::SQS::Client is not entirely
+    # thread-safe - or at least not it's HTTP client part).
     box = Sqewer::ConnectionMessagebox.new(connection)
     
     job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
@@ -60,10 +65,8 @@ class Sqewer::Isolator
     
     delta = Time.now - t
     logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
-  rescue => e
-    raise e
   ensure
     n_flushed = box.flush!
-    logger.info { "[worker] Flushed %d messages" % n_flushed } if n_flushed.nonzero?
+    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
   end
 end

data/lib/sqewer/state_lock.rb

@@ -0,0 +1,24 @@
+require 'delegate'
+require 'thread'
+
+# Controls the state of the Worker object, and wraps it's state transitions
+# with a Mutex.
+class Sqewer::StateLock < SimpleDelegator
+  def initialize
+    @m = Mutex.new
+    m = VeryTinyStateMachine.new(:stopped)
+    m.permit_state :starting, :running, :stopping, :stopped, :failed
+    m.permit_transition :stopped => :starting, :starting => :running
+    m.permit_transition :running => :stopping, :stopping => :stopped
+    m.permit_transition :starting => :failed # Failed to start
+    __setobj__(m)
+  end
+  
+  def in_state?(some_state)
+    @m.synchronize { __getobj__.in_state?(some_state) }
+  end
+    
+  def transition!(to_state)
+    @m.synchronize { __getobj__.transition!(to_state) }
+  end
+end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '4.0.0'
+  VERSION = '4.0.1'
 end

data/lib/sqewer/worker.rb

@@ -74,10 +74,7 @@ class Sqewer::Worker
     
     @execution_counter = Sqewer::AtomicCounter.new
     
-    @state = VeryTinyStateMachine.new(:stopped)
-    @state.permit_state :starting, :running, :stopping, :stopped, :failed
-    @state.permit_transition :stopped => :starting, :starting => :running, :running => :stopping, :stopping => :stopped
-    @state.permit_transition :starting => :failed # Failed to start
+    @state = Sqewer::StateLock.new
   end
   
   # Start listening on the queue, spin up a number of consumer threads that will execute the jobs.
@@ -117,9 +114,8 @@ class Sqewer::Worker
             Thread.pass
           end
         else
-          @logger.debug { "[worker] Suspending poller (%d items buffered)" % @execution_queue.length }
-          sleep 1
-          Thread.pass
+          @logger.debug { "[worker] Cache is full (%d items), postponing receive" % @execution_queue.length }
+          sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
         end 
       end
     end
@@ -147,7 +143,7 @@ class Sqewer::Worker
       break if n_live.zero?
       
       n_dead = @threads.length - n_live
-      @logger.info {"Waiting on threads to terminate, %d still alive, %d quit" % [n_live, n_dead] }
+      @logger.info { '[worker] Waiting on threads to terminate, %d still alive, %d quit' % [n_live, n_dead] }
       
       sleep 2
     end
@@ -188,9 +184,8 @@ class Sqewer::Worker
     handle_message(message)
   rescue ThreadError # Queue is empty
     sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
-    Thread.pass
   rescue => e # anything else, at or below StandardError that does not need us to quit
-    @logger.error { "[worker] Failed %s... with %s: %s" % [message.inspect[0..64], e.class, e.message] }
+    @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..32], e.class, e.message] }
     e.backtrace.each { |s| @logger.error{"\t#{s}"} }
   end
 end

data/spec/sqewer/cli_spec.rb

@@ -12,7 +12,7 @@ describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
       stderr = Tempfile.new('worker-stderr')
       stderr.sync = true
       
-      pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
+      pid = fork { $stderr.reopen(stderr); $stderr.sync = true; exec("ruby #{__dir__}/cli_app.rb") }
   
       Thread.new do
         20.times do
@@ -39,7 +39,7 @@ describe Sqewer::CLI, :sqs => true, :wait => {timeout: 120} do
       stderr = Tempfile.new('worker-stderr')
       stderr.sync = true
     
-      pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
+      pid = fork { $stderr.reopen(stderr); $stderr.sync; exec("ruby #{__dir__}/cli_app.rb") }
   
       Thread.new do
         20.times do

data/sqewer.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sqewer 4.0.0 ruby lib
+# stub: sqewer 4.0.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sqewer"
-  s.version = "4.0.0"
+  s.version = "4.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "lib/sqewer/null_logger.rb",
     "lib/sqewer/serializer.rb",
     "lib/sqewer/simple_job.rb",
+    "lib/sqewer/state_lock.rb",
     "lib/sqewer/submitter.rb",
     "lib/sqewer/version.rb",
     "lib/sqewer/worker.rb",
@@ -67,7 +68,7 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<aws-sdk>, ["~> 2"])
-      s.add_runtime_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+      s.add_runtime_dependency(%q<very_tiny_state_machine>, [">= 0"])
       s.add_runtime_dependency(%q<hash_tools>, [">= 0"])
       s.add_runtime_dependency(%q<exceptional_fork>, [">= 0"])
       s.add_development_dependency(%q<ks>, [">= 0"])
@@ -81,7 +82,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
     else
       s.add_dependency(%q<aws-sdk>, ["~> 2"])
-      s.add_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+      s.add_dependency(%q<very_tiny_state_machine>, [">= 0"])
       s.add_dependency(%q<hash_tools>, [">= 0"])
       s.add_dependency(%q<exceptional_fork>, [">= 0"])
       s.add_dependency(%q<ks>, [">= 0"])
@@ -96,7 +97,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<aws-sdk>, ["~> 2"])
-    s.add_dependency(%q<very_tiny_state_machine>, ["~> 1"])
+    s.add_dependency(%q<very_tiny_state_machine>, [">= 0"])
     s.add_dependency(%q<hash_tools>, [">= 0"])
     s.add_dependency(%q<exceptional_fork>, [">= 0"])
     s.add_dependency(%q<ks>, [">= 0"])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -28,16 +28,16 @@ dependencies:
   name: very_tiny_state_machine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: hash_tools
   requirement: !ruby/object:Gem::Requirement
@@ -221,6 +221,7 @@ files:
 - lib/sqewer/null_logger.rb
 - lib/sqewer/serializer.rb
 - lib/sqewer/simple_job.rb
+- lib/sqewer/state_lock.rb
 - lib/sqewer/submitter.rb
 - lib/sqewer/version.rb
 - lib/sqewer/worker.rb