toiler 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1411df909332ded188599a79d720d618e46bf5bd
-  data.tar.gz: 1e665ab6d968eaa0a85955202e5d78e8fedd99a0
+  metadata.gz: 0ece2dedcdd070ea808c72bf4c9dc2dfa5ed40a2
+  data.tar.gz: 860bb03a04d9db9f759f9be0f1ef9927086490e0
 SHA512:
-  metadata.gz: b2fa690e79c8a88c53ef0e4df2445f57f1eb5212db96b3bb61904d035eed5bcefef868fdf37c949fe650e70875b210acfbcfe836bd7c2fdf2d1a3accf14bc96f
-  data.tar.gz: bf9f7db7c915ed650d81597ac0c0eb74e0a65215fdad13e0c82d7a843ff9d5d76ab4d7978882b61a01362237c2b358c8cbc7643534f7c1e0a1c719997d3141fc
+  metadata.gz: 9547e8024e54d8ea73c305010bf89980e002c1293840b49010bd6241cab7fa1f4d2f2de373acfccf56775baead38559c86220310fde6a5917db97950ef533309
+  data.tar.gz: 7bb8c7bc6b85d49c90537c7267a5bad510f10d3ac22fd809b014e48f199c2911a35170f76bb1346c2e4b2b7081c0d8125e36db79f312e4bf4c959f62d5019a92

data/lib/toiler/fetcher.rb

@@ -10,13 +10,16 @@ module Toiler
     finalizer :shutdown
 
     def initialize(queue, client = nil)
+      debug "Initializing Fetcher for queue #{queue}..."
       @queue = Queue.new queue, client
       @wait = Toiler.options[:wait] || 20
       @batch = Toiler.worker_class_registry[queue].batch?
       async.poll_messages
+      debug "Finished initializing Fetcher for queue #{queue}"
     end
 
     def shutdown
+      debug "Fetcher #{queue.name} shutting down..."
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
@@ -30,8 +33,11 @@ module Toiler
       loop do
         count = Toiler.manager.free_processors queue.name
         options[:max_number_of_messages] = (batch || count > FETCH_LIMIT) ? FETCH_LIMIT : count
+        debug "Fetcher #{queue.name} retreiving messages with options: #{options.inspect}..."
         msgs = queue.receive_messages options
-        Toiler.manager.assign_messages queue.name, msgs unless msgs.empty?
+        debug "Fetcher #{queue.name} retreived #{msgs.count} messages..."
+        next if msgs.empty?
+        Toiler.manager.assign_messages queue.name, msgs
         Toiler.manager.wait_for_available_processors queue.name
       end
     end

data/lib/toiler/manager.rb

@@ -16,19 +16,23 @@ module Toiler
     end
 
     def init
+      debug 'Initializing manager...'
       @queues = Toiler.worker_class_registry
       @client = ::Aws::SQS::Client.new
       init_workers
       init_conditions
       pool_processors
       supervise_fetchers
+      debug 'Finished initializing manager...'
     end
 
     def shutdown
+      debug 'Manager shutting down...'
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
     def stop
+      debug 'Manager stopping...'
       terminate_fetchers
       terminate_processors
     end
@@ -88,6 +92,7 @@ module Toiler
     end
 
     def assign_messages(queue, messages)
+      debug "Manager assigning #{messages.count} for queue #{queue}"
       processor_pool = Toiler.processor_pool(queue)
       if batch? queue
         processor_pool.async.process(queue, messages)
@@ -96,6 +101,7 @@ module Toiler
           processor_pool.async.process(queue, m)
         end
       end
+      debug "Manager finished assigning #{messages.count} for queue #{queue}"
     end
 
     def wait_for_available_processors(queue)

data/lib/toiler/processor.rb

@@ -11,21 +11,21 @@ module Toiler
     finalizer :shutdown
 
     def initialize(queue)
+      debug "Initializing Processor for queue #{queue}"
       @queue = queue
-      async.init
-    end
-
-    def init
       @scheduler = Scheduler.supervise.actors.first
       processor_finished
+      debug "Finished initializing Processor for queue #{queue}"
     end
 
     def shutdown
+      debug "Processor for queue #{queue} shutting down..."
       scheduler.terminate if scheduler && scheduler.alive?
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
     def process(queue, sqs_msg)
+      debug "Processor #{queue} begins processing..."
       exclusive do
         worker = Toiler.worker_registry[queue]
         timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
@@ -36,11 +36,11 @@ module Toiler
           sqs_msg.delete if worker.class.auto_delete?
         ensure
           timer.cancel if timer
-          ::ActiveRecord::Base.clear_active_connections!
+          ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
         end
       end
-    ensure
       processor_finished
+      debug "Processor #{queue} finishes processing..."
     end
 
     def processor_finished
@@ -52,11 +52,12 @@ module Toiler
     def auto_visibility_timeout(queue, sqs_msg, worker_class)
       return unless worker_class.auto_visibility_timeout?
       queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
-      block = lambda do |msg, visibility_timeout|
+      block = lambda do |msg, visibility_timeout, q|
+        debug "Processor #{q} updating visibility_timeout..."
         msg.visibility_timeout = visibility_timeout
       end
 
-      scheduler.custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, block)
+      scheduler.custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
     end
 
     def get_body(worker_class, sqs_msg)

data/lib/toiler/version.rb

@@ -1,3 +1,3 @@
 module Toiler
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Sebastian Schepens