toiler 0.3.1.beta1 → 0.3.1.beta2

data/lib/toiler/actor/fetcher.rb

@@ -1,104 +1,120 @@
-require 'toiler/actor/utils/actor_logging'
-require 'toiler/aws/queue'
-
-module Toiler
-  module Actor
-    # Actor polling for messages only when processors are ready, otherwise idle
-    class Fetcher < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      FETCH_LIMIT = 10
-
-      attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :scheduled, :executing
-
-      def initialize(queue, client)
-        debug "Initializing Fetcher for queue #{queue}..."
-        @queue = Toiler::Aws::Queue.new queue, client
-        @wait = Toiler.options[:wait] || 20
-        @free_processors = Concurrent::AtomicFixnum.new(0)
-        @batch = Toiler.worker_class_registry[queue].batch?
-        @visibility_timeout = @queue.visibility_timeout
-        @scheduled = Concurrent::AtomicBoolean.new
-        @executing = Concurrent::AtomicBoolean.new
-        debug "Finished initializing Fetcher for queue #{queue}"
-      end
-
-      def default_executor
-        Concurrent.global_io_executor
-      end
-
-      def on_message(msg)
-        method, *args = msg
-        send(method, *args)
-      rescue StandardError => e
-        error "Fetcher #{queue.name} raised exception #{e.class}"
-      end
-
-      def executing?
-        executing.value
-      end
-
-      def scheduled?
-        scheduled.value
-      end
-
-      def get_free_processors
-        free_processors.value
-      end
-
-      private
-
-      def batch?
-        @batch
-      end
-
-      def processor_finished
-        debug "Fetcher #{queue.name} received processor finished signal..."
-        free_processors.increment
-        schedule_poll
-      end
-
-      def max_messages
-        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
-      end
-
-      def poll_future
-        Concurrent.future do
-          @executing.make_true
-          queue.receive_messages message_attribute_names: %w(All),
-                                 wait_time_seconds: wait,
-                                 max_number_of_messages: max_messages
-          @executing.make_false
-        end
-      end
-
-      def poll_messages
-        poll_future.on_completion! do |_success, msgs|
-          tell [:assign_messages, msgs] unless msgs.nil? || msgs.empty?
-          scheduled.make_false
-          tell :schedule_poll
-        end
-      end
-
-      def schedule_poll
-        return unless free_processors.value > 0 && scheduled.make_true
-        debug "Fetcher #{queue.name} scheduling polling..."
-        tell :poll_messages
-      end
-
-      def processor_pool
-        @processor_pool ||= Toiler.processor_pool queue.name
-      end
-
-      def assign_messages(messages)
-        messages = [messages] if batch?
-        messages.each do |m|
-          processor_pool.tell [:process, visibility_timeout, m]
-          free_processors.decrement
-        end
-        debug "Fetcher #{queue.name} assigned #{messages.count} messages"
-      end
-    end
-  end
-end
+require 'toiler/actor/utils/actor_logging'
+require 'toiler/aws/queue'
+
+module Toiler
+  module Actor
+    # Actor polling for messages only when processors are ready, otherwise idle
+    class Fetcher < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      FETCH_LIMIT = 10
+
+      attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
+                    :scheduled, :executing
+
+      def initialize(queue, client)
+        debug "Initializing Fetcher for queue #{queue}..."
+        @queue = Toiler::Aws::Queue.new queue, client
+        @wait = Toiler.options[:wait] || 20
+        @free_processors = Concurrent::AtomicFixnum.new(0)
+        @batch = Toiler.worker_class_registry[queue].batch?
+        @visibility_timeout = @queue.visibility_timeout
+        @scheduled = Concurrent::AtomicBoolean.new
+        @executing = Concurrent::AtomicBoolean.new
+        @polling = Concurrent::AtomicBoolean.new
+        debug "Finished initializing Fetcher for queue #{queue}"
+      end
+
+      def default_executor
+        Concurrent.global_io_executor
+      end
+
+      def on_message(msg)
+        executing.make_true
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Fetcher #{queue.name} raised exception #{e.class}"
+      ensure
+        executing.make_false
+      end
+
+      def executing?
+        executing.value
+      end
+
+      def polling?
+        polling.value
+      end
+
+      def scheduled?
+        scheduled.value
+      end
+
+      def get_free_processors
+        free_processors.value
+      end
+
+      private
+
+      def batch?
+        @batch
+      end
+
+      def processor_started
+        debug "Fetcher #{queue.name} received processor started signal..."
+        free_processors.decrement
+      end
+
+      def processor_finished
+        debug "Fetcher #{queue.name} received processor finished signal..."
+        free_processors.increment
+        schedule_poll
+      end
+
+      def max_messages
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
+      end
+
+      def poll_future
+        Concurrent.future do
+          queue.receive_messages message_attribute_names: %w(All),
+                                 wait_time_seconds: wait,
+                                 max_number_of_messages: max_messages
+        end
+      end
+
+      def poll_messages
+        polling.make_true
+        poll_future.on_completion! do |success, msgs, error|
+          polling.make_false
+          scheduled.make_false
+          if success && !msgs.nil? && !msgs.empty?
+            tell [:assign_messages, msgs]
+          else
+            tell :schedule_poll
+          end
+        end
+      end
+
+      def schedule_poll
+        return unless free_processors.value > 0 && scheduled.make_true
+        debug "Fetcher #{queue.name} scheduling polling..."
+        tell :poll_messages
+      end
+
+      def processor_pool
+        @processor_pool ||= Toiler.processor_pool queue.name
+      end
+
+      def assign_messages(messages)
+        messages = [messages] if batch?
+        messages.each do |m|
+          processor_pool.tell [:process, visibility_timeout, m]
+        end
+        debug "Fetcher #{queue.name} assigned #{messages.count} messages"
+        tell :schedule_poll
+      end
+    end
+  end
+end

data/lib/toiler/actor/processor.rb

@@ -1,119 +1,128 @@
-require 'json'
-require 'toiler/actor/utils/actor_logging'
-
-module Toiler
-  module Actor
-    # Responsible for processing sqs messages and notifying Fetcher when done
-    class Processor < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      attr_accessor :queue, :worker, :fetcher, :body_parser,
-                    :extend_callback, :executing, :thread
-
-      def initialize(queue)
-        @queue = queue
-        @worker = Toiler.worker_class_registry[queue].new
-        @fetcher = Toiler.fetcher queue
-        @executing = Concurrent::AtomicBoolean.new
-        @thread = nil
-        init_options
-        processor_finished
-      end
-
-      def default_executor
-        Concurrent.global_io_executor
-      end
-
-      def on_message(msg)
-        method, *args = msg
-        send(method, *args)
-      rescue StandardError => e
-        error "Processor #{queue} failed processing, reason: #{e.class}\n#{e.backtrace.join("\n")}"
-      end
-
-      def executing?
-        executing.value
-      end
-
-      private
-
-      def init_options
-        @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
-        @auto_delete = @worker.class.auto_delete?
-        toiler_options = @worker.class.toiler_options
-        @body_parser = toiler_options[:parser]
-        @extend_callback = toiler_options[:on_visibility_extend]
-      end
-
-      def auto_visibility_timeout?
-        @auto_visibility_timeout
-      end
-
-      def auto_delete?
-        @auto_delete
-      end
-
-      def process(visibility, sqs_msg)
-        executing.make_true
-        @thread = Thread.current
-        debug "Processor #{queue} begins processing..."
-        body = get_body(sqs_msg)
-        timer = visibility_extender visibility, sqs_msg, body, &extend_callback
-
-        debug "Worker #{queue} starts performing..."
-        worker.perform sqs_msg, body
-        debug "Worker #{queue} finishes performing..."
-        sqs_msg.delete if auto_delete?
-      ensure
-        process_cleanup timer
-        executing.make_false
-        @thread = nil
-      end
-
-      def process_cleanup(timer)
-        debug "Processor #{queue} starts cleanup after perform..."
-        timer.shutdown if timer
-        ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
-        processor_finished
-        debug "Processor #{queue} finished cleanup after perform..."
-      end
-
-      def processor_finished
-        fetcher.tell :processor_finished
-      end
-
-      def visibility_extender(queue_visibility, sqs_msg, body)
-        return unless auto_visibility_timeout?
-        interval = [1,queue_visibility/3].max
-        Concurrent::TimerTask.execute execution_interval: interval,
-                                      timeout_interval: interval do
-          begin
-            sqs_msg.visibility_timeout = queue_visibility
-            yield sqs_msg, body if block_given?
-          rescue StandardError => e
-            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-
-      def get_body(sqs_msg)
-        if sqs_msg.is_a? Array
-          sqs_msg.map { |m| parse_body m }
-        else
-          parse_body sqs_msg
-        end
-      end
-
-      def parse_body(sqs_msg)
-        case body_parser
-        when :json then JSON.parse sqs_msg.body
-        when Proc then body_parser.call sqs_msg
-        when :text, nil then sqs_msg.body
-        else body_parser.load sqs_msg.body
-        end
-      rescue => e
-        raise "Error parsing the message body: #{e.message}"
-      end
-    end
-  end
-end
+require 'json'
+require 'toiler/actor/utils/actor_logging'
+
+module Toiler
+  module Actor
+    # Responsible for processing sqs messages and notifying Fetcher when done
+    class Processor < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      attr_accessor :queue, :worker, :fetcher, :body_parser,
+                    :extend_callback, :executing, :thread
+
+      def initialize(queue)
+        @queue = queue
+        @worker = Toiler.worker_class_registry[queue].new
+        @fetcher = Toiler.fetcher queue
+        @executing = Concurrent::AtomicBoolean.new
+        @thread = nil
+        init_options
+        processor_finished
+      end
+
+      def default_executor
+        Concurrent.global_io_executor
+      end
+
+      def on_message(msg)
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Processor #{queue} failed processing, reason: #{e.class}\n#{e.backtrace.join("\n")}"
+      end
+
+      def executing?
+        executing.value
+      end
+
+      private
+
+      def init_options
+        @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
+        @auto_delete = @worker.class.auto_delete?
+        toiler_options = @worker.class.toiler_options
+        @body_parser = toiler_options[:parser]
+        @extend_callback = toiler_options[:on_visibility_extend]
+      end
+
+      def auto_visibility_timeout?
+        @auto_visibility_timeout
+      end
+
+      def auto_delete?
+        @auto_delete
+      end
+
+      def process(visibility, sqs_msg)
+        process_init
+        body = get_body(sqs_msg)
+        timer = visibility_extender visibility, sqs_msg, body, &extend_callback
+
+        debug "Worker #{queue} starts performing..."
+        worker.perform sqs_msg, body
+        debug "Worker #{queue} finishes performing..."
+        sqs_msg.delete if auto_delete?
+      ensure
+        process_cleanup timer
+      end
+
+      def process_init
+        processor_started
+        @executing.make_true
+        @thread = Thread.current
+        debug "Processor #{queue} begins processing..."
+      end
+
+      def process_cleanup(timer)
+        debug "Processor #{queue} starts cleanup after perform..."
+        timer.shutdown if timer
+        ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
+        processor_finished
+        @executing.make_false
+        @thread = nil
+        debug "Processor #{queue} finished cleanup after perform..."
+      end
+
+      def processor_finished
+        fetcher.tell :processor_finished
+      end
+
+      def processor_started
+        fetcher.tell :processor_started
+      end
+
+      def visibility_extender(queue_visibility, sqs_msg, body)
+        return unless auto_visibility_timeout?
+        interval = [1,queue_visibility/3].max
+        Concurrent::TimerTask.execute execution_interval: interval,
+                                      timeout_interval: interval do
+          begin
+            sqs_msg.visibility_timeout = queue_visibility
+            yield sqs_msg, body if block_given?
+          rescue StandardError => e
+            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      def get_body(sqs_msg)
+        if sqs_msg.is_a? Array
+          sqs_msg.map { |m| parse_body m }
+        else
+          parse_body sqs_msg
+        end
+      end
+
+      def parse_body(sqs_msg)
+        case body_parser
+        when :json then JSON.parse sqs_msg.body
+        when Proc then body_parser.call sqs_msg
+        when :text, nil then sqs_msg.body
+        else body_parser.load sqs_msg.body
+        end
+      rescue => e
+        raise "Error parsing the message body: #{e.message}"
+      end
+    end
+  end
+end

data/lib/toiler/actor/supervisor.rb

@@ -1,55 +1,55 @@
-require 'toiler/actor/fetcher'
-require 'toiler/actor/processor'
-
-module Toiler
-  module Actor
-    # Actor that starts and supervises Toiler's actors
-    class Supervisor < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      attr_accessor :client
-
-      def initialize
-        @client = ::Aws::SQS::Client.new
-        spawn_fetchers
-        spawn_processors
-      end
-
-      def on_message(_msg)
-        pass
-      end
-
-      def queues
-        Toiler.worker_class_registry
-      end
-
-      def spawn_fetchers
-        queues.each do |queue, _klass|
-          begin
-            fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
-                                            supervise: true, args: [queue, client]
-            Toiler.set_fetcher queue, fetcher
-          rescue StandardError => e
-            error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-
-      def spawn_processors
-        queues.each do |queue, klass|
-          name = "processor_pool_#{queue}".to_sym
-          count = klass.concurrency
-          begin
-            pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
-                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
-                                       supervise: true, args: [queue]
-            end
-            Toiler.set_processor_pool queue, pool
-          rescue StandardError => e
-            error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-    end
-  end
-end
+require 'toiler/actor/fetcher'
+require 'toiler/actor/processor'
+
+module Toiler
+  module Actor
+    # Actor that starts and supervises Toiler's actors
+    class Supervisor < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      attr_accessor :client
+
+      def initialize
+        @client = ::Aws::SQS::Client.new
+        spawn_fetchers
+        spawn_processors
+      end
+
+      def on_message(_msg)
+        pass
+      end
+
+      def queues
+        Toiler.worker_class_registry
+      end
+
+      def spawn_fetchers
+        queues.each do |queue, _klass|
+          begin
+            fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
+                                            supervise: true, args: [queue, client]
+            Toiler.set_fetcher queue, fetcher
+          rescue StandardError => e
+            error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      def spawn_processors
+        queues.each do |queue, klass|
+          name = "processor_pool_#{queue}".to_sym
+          count = klass.concurrency
+          begin
+            pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
+                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                       supervise: true, args: [queue]
+            end
+            Toiler.set_processor_pool queue, pool
+          rescue StandardError => e
+            error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toiler/actor/utils/actor_logging.rb

@@ -1,28 +1,28 @@
-module Toiler
-  module Actor
-    module Utils
-      # Provides helper methods for logging
-      module ActorLogging
-        def error(msg)
-          log Logger::Severity::ERROR,  msg
-        end
-
-        def info(msg)
-          log Logger::Severity::INFO, msg
-        end
-
-        def debug(msg)
-          log Logger::Severity::DEBUG,  msg
-        end
-
-        def warn(msg)
-          log Logger::Severity::WARN, smsg
-        end
-
-        def fatal(msg)
-          log Logger::Severity::FATAL,  msg
-        end
-      end
-    end
-  end
-end
+module Toiler
+  module Actor
+    module Utils
+      # Provides helper methods for logging
+      module ActorLogging
+        def error(msg)
+          log Logger::Severity::ERROR,  msg
+        end
+
+        def info(msg)
+          log Logger::Severity::INFO, msg
+        end
+
+        def debug(msg)
+          log Logger::Severity::DEBUG,  msg
+        end
+
+        def warn(msg)
+          log Logger::Severity::WARN, smsg
+        end
+
+        def fatal(msg)
+          log Logger::Severity::FATAL,  msg
+        end
+      end
+    end
+  end
+end