toiler 0.3.0 → 0.3.1.beta1

data/Rakefile

@@ -1 +1 @@
-require 'bundler/gem_tasks'
+require 'bundler/gem_tasks'

data/bin/toiler

@@ -1,12 +1,12 @@
-#!/usr/bin/env ruby
-
-require 'toiler'
-
-begin
-  Toiler::CLI.instance.run(ARGV)
-rescue => e
-  raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
-  exit 1
-end
+#!/usr/bin/env ruby
+
+require 'toiler'
+
+begin
+  Toiler::CLI.instance.run(ARGV)
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/toiler.rb

@@ -1,55 +1,55 @@
-require 'aws-sdk'
-require 'toiler/utils/environment_loader'
-require 'toiler/utils/logging'
-require 'toiler/utils/argument_parser'
-require 'toiler/worker'
-require 'toiler/cli'
-require 'toiler/version'
-
-# Main module
-module Toiler
-  @worker_class_registry = {}
-  @options = {
-    aws: {}
-  }
-  @fetchers = {}
-  @processor_pools = {}
-
-  attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
-  module_function :worker_class_registry, :options, :fetchers, :processor_pools
-
-  module_function
-
-  def logger
-    Toiler::Utils::Logging.logger
-  end
-
-  def queues
-    worker_class_registry.keys
-  end
-
-  def fetcher(queue)
-    fetchers["fetcher_#{queue}".to_sym]
-  end
-
-  def set_fetcher(queue, val)
-    fetchers["fetcher_#{queue}".to_sym] = val
-  end
-
-  def processor_pool(queue)
-    processor_pools["processor_pool_#{queue}".to_sym]
-  end
-
-  def set_processor_pool(queue, val)
-    processor_pools["processor_pool_#{queue}".to_sym] = val
-  end
-
-  def default_options
-    {
-      auto_visibility_timeout: false,
-      concurrency: 1,
-      auto_delete: false,
-      batch: false
-    }
-  end
-end
+require 'aws-sdk'
+require 'toiler/utils/environment_loader'
+require 'toiler/utils/logging'
+require 'toiler/utils/argument_parser'
+require 'toiler/worker'
+require 'toiler/cli'
+require 'toiler/version'
+
+# Main module
+module Toiler
+  @worker_class_registry = {}
+  @options = {
+    aws: {}
+  }
+  @fetchers = {}
+  @processor_pools = {}
+
+  attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
+  module_function :worker_class_registry, :options, :fetchers, :processor_pools
+
+  module_function
+
+  def logger
+    Toiler::Utils::Logging.logger
+  end
+
+  def queues
+    worker_class_registry.keys
+  end
+
+  def fetcher(queue)
+    fetchers["fetcher_#{queue}".to_sym]
+  end
+
+  def set_fetcher(queue, val)
+    fetchers["fetcher_#{queue}".to_sym] = val
+  end
+
+  def processor_pool(queue)
+    processor_pools["processor_pool_#{queue}".to_sym]
+  end
+
+  def set_processor_pool(queue, val)
+    processor_pools["processor_pool_#{queue}".to_sym] = val
+  end
+
+  def default_options
+    {
+      auto_visibility_timeout: false,
+      concurrency: 1,
+      auto_delete: false,
+      batch: false
+    }
+  end
+end

data/lib/toiler/actor/fetcher.rb

@@ -1,89 +1,104 @@
-require 'toiler/actor/utils/actor_logging'
-require 'toiler/aws/queue'
-
-module Toiler
-  module Actor
-    # Actor polling for messages only when processors are ready, otherwise idle
-    class Fetcher < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      FETCH_LIMIT = 10
-
-      attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :scheduled
-
-      def initialize(queue, client)
-        debug "Initializing Fetcher for queue #{queue}..."
-        @queue = Toiler::Aws::Queue.new queue, client
-        @wait = Toiler.options[:wait] || 20
-        @free_processors = Concurrent::AtomicFixnum.new(0)
-        @batch = Toiler.worker_class_registry[queue].batch?
-        @visibility_timeout = @queue.visibility_timeout
-        @scheduled = Concurrent::AtomicBoolean.new
-        debug "Finished initializing Fetcher for queue #{queue}"
-      end
-
-      def default_executor
-        Concurrent.global_io_executor
-      end
-
-      def on_message(msg)
-        method, *args = msg
-        send(method, *args)
-      rescue StandardError => e
-        error "Fetcher #{queue.name} raised exception #{e.class}"
-      end
-
-      private
-
-      def batch?
-        @batch
-      end
-
-      def processor_finished
-        debug "Fetcher #{queue.name} received processor finished signal..."
-        free_processors.increment
-        schedule_poll
-      end
-
-      def max_messages
-        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
-      end
-
-      def poll_future
-        Concurrent.future do
-          queue.receive_messages message_attribute_names: %w(All),
-                                 wait_time_seconds: wait,
-                                 max_number_of_messages: max_messages
-        end
-      end
-
-      def poll_messages
-        poll_future.on_completion! do |_success, msgs|
-          tell [:assign_messages, msgs] unless msgs.nil? || msgs.empty?
-          scheduled.make_false
-          tell :schedule_poll
-        end
-      end
-
-      def schedule_poll
-        return unless free_processors.value > 0 && scheduled.make_true
-        debug "Fetcher #{queue.name} scheduling polling..."
-        tell :poll_messages
-      end
-
-      def processor_pool
-        @processor_pool ||= Toiler.processor_pool queue.name
-      end
-
-      def assign_messages(messages)
-        messages = [messages] if batch?
-        messages.each do |m|
-          processor_pool.tell [:process, visibility_timeout, m]
-          free_processors.decrement
-        end
-        debug "Fetcher #{queue.name} assigned #{messages.count} messages"
-      end
-    end
-  end
-end
+require 'toiler/actor/utils/actor_logging'
+require 'toiler/aws/queue'
+
+module Toiler
+  module Actor
+    # Actor polling for messages only when processors are ready, otherwise idle
+    class Fetcher < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      FETCH_LIMIT = 10
+
+      attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
+                    :scheduled, :executing
+
+      def initialize(queue, client)
+        debug "Initializing Fetcher for queue #{queue}..."
+        @queue = Toiler::Aws::Queue.new queue, client
+        @wait = Toiler.options[:wait] || 20
+        @free_processors = Concurrent::AtomicFixnum.new(0)
+        @batch = Toiler.worker_class_registry[queue].batch?
+        @visibility_timeout = @queue.visibility_timeout
+        @scheduled = Concurrent::AtomicBoolean.new
+        @executing = Concurrent::AtomicBoolean.new
+        debug "Finished initializing Fetcher for queue #{queue}"
+      end
+
+      def default_executor
+        Concurrent.global_io_executor
+      end
+
+      def on_message(msg)
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Fetcher #{queue.name} raised exception #{e.class}"
+      end
+
+      def executing?
+        executing.value
+      end
+
+      def scheduled?
+        scheduled.value
+      end
+
+      def get_free_processors
+        free_processors.value
+      end
+
+      private
+
+      def batch?
+        @batch
+      end
+
+      def processor_finished
+        debug "Fetcher #{queue.name} received processor finished signal..."
+        free_processors.increment
+        schedule_poll
+      end
+
+      def max_messages
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
+      end
+
+      def poll_future
+        Concurrent.future do
+          @executing.make_true
+          queue.receive_messages message_attribute_names: %w(All),
+                                 wait_time_seconds: wait,
+                                 max_number_of_messages: max_messages
+          @executing.make_false
+        end
+      end
+
+      def poll_messages
+        poll_future.on_completion! do |_success, msgs|
+          tell [:assign_messages, msgs] unless msgs.nil? || msgs.empty?
+          scheduled.make_false
+          tell :schedule_poll
+        end
+      end
+
+      def schedule_poll
+        return unless free_processors.value > 0 && scheduled.make_true
+        debug "Fetcher #{queue.name} scheduling polling..."
+        tell :poll_messages
+      end
+
+      def processor_pool
+        @processor_pool ||= Toiler.processor_pool queue.name
+      end
+
+      def assign_messages(messages)
+        messages = [messages] if batch?
+        messages.each do |m|
+          processor_pool.tell [:process, visibility_timeout, m]
+          free_processors.decrement
+        end
+        debug "Fetcher #{queue.name} assigned #{messages.count} messages"
+      end
+    end
+  end
+end

data/lib/toiler/actor/processor.rb

@@ -1,110 +1,119 @@
-require 'json'
-require 'toiler/actor/utils/actor_logging'
-
-module Toiler
-  module Actor
-    # Responsible for processing sqs messages and notifying Fetcher when done
-    class Processor < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      attr_accessor :queue, :worker, :fetcher, :body_parser,
-                    :extend_callback
-
-      def initialize(queue)
-        @queue = queue
-        @worker = Toiler.worker_class_registry[queue].new
-        @fetcher = Toiler.fetcher queue
-        init_options
-        processor_finished
-      end
-
-      def default_executor
-        Concurrent.global_io_executor
-      end
-
-      def on_message(msg)
-        method, *args = msg
-        send(method, *args)
-      rescue StandardError => e
-        error "Processor #{queue} failed processing, reason: #{e.class}\n#{e.backtrace.join("\n")}"
-        raise e
-      end
-
-      private
-
-      def init_options
-        @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
-        @auto_delete = @worker.class.auto_delete?
-        toiler_options = @worker.class.toiler_options
-        @body_parser = toiler_options[:parser]
-        @extend_callback = toiler_options[:on_visibility_extend]
-      end
-
-      def auto_visibility_timeout?
-        @auto_visibility_timeout
-      end
-
-      def auto_delete?
-        @auto_delete
-      end
-
-      def process(visibility, sqs_msg)
-        debug "Processor #{queue} begins processing..."
-        body = get_body(sqs_msg)
-        timer = visibility_extender visibility, sqs_msg, body, &extend_callback
-
-        debug "Worker #{queue} starts performing..."
-        worker.perform sqs_msg, body
-        debug "Worker #{queue} finishes performing..."
-        sqs_msg.delete if auto_delete?
-      ensure
-        process_cleanup timer
-      end
-
-      def process_cleanup(timer)
-        debug "Processor #{queue} starts cleanup after perform..."
-        timer.shutdown if timer
-        ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
-        processor_finished
-        debug "Processor #{queue} finished cleanup after perform..."
-      end
-
-      def processor_finished
-        fetcher.tell :processor_finished
-      end
-
-      def visibility_extender(queue_visibility, sqs_msg, body)
-        return unless auto_visibility_timeout?
-        interval = [1,queue_visibility/3].max
-        Concurrent::TimerTask.execute execution_interval: interval,
-                                      timeout_interval: interval do
-          begin
-            sqs_msg.visibility_timeout = queue_visibility
-            yield sqs_msg, body if block_given?
-          rescue StandardError => e
-            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-
-      def get_body(sqs_msg)
-        if sqs_msg.is_a? Array
-          sqs_msg.map { |m| parse_body m }
-        else
-          parse_body sqs_msg
-        end
-      end
-
-      def parse_body(sqs_msg)
-        case body_parser
-        when :json then JSON.parse sqs_msg.body
-        when Proc then body_parser.call sqs_msg
-        when :text, nil then sqs_msg.body
-        else body_parser.load sqs_msg.body
-        end
-      rescue => e
-        raise "Error parsing the message body: #{e.message}"
-      end
-    end
-  end
-end
+require 'json'
+require 'toiler/actor/utils/actor_logging'
+
+module Toiler
+  module Actor
+    # Responsible for processing sqs messages and notifying Fetcher when done
+    class Processor < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      attr_accessor :queue, :worker, :fetcher, :body_parser,
+                    :extend_callback, :executing, :thread
+
+      def initialize(queue)
+        @queue = queue
+        @worker = Toiler.worker_class_registry[queue].new
+        @fetcher = Toiler.fetcher queue
+        @executing = Concurrent::AtomicBoolean.new
+        @thread = nil
+        init_options
+        processor_finished
+      end
+
+      def default_executor
+        Concurrent.global_io_executor
+      end
+
+      def on_message(msg)
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Processor #{queue} failed processing, reason: #{e.class}\n#{e.backtrace.join("\n")}"
+      end
+
+      def executing?
+        executing.value
+      end
+
+      private
+
+      def init_options
+        @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
+        @auto_delete = @worker.class.auto_delete?
+        toiler_options = @worker.class.toiler_options
+        @body_parser = toiler_options[:parser]
+        @extend_callback = toiler_options[:on_visibility_extend]
+      end
+
+      def auto_visibility_timeout?
+        @auto_visibility_timeout
+      end
+
+      def auto_delete?
+        @auto_delete
+      end
+
+      def process(visibility, sqs_msg)
+        executing.make_true
+        @thread = Thread.current
+        debug "Processor #{queue} begins processing..."
+        body = get_body(sqs_msg)
+        timer = visibility_extender visibility, sqs_msg, body, &extend_callback
+
+        debug "Worker #{queue} starts performing..."
+        worker.perform sqs_msg, body
+        debug "Worker #{queue} finishes performing..."
+        sqs_msg.delete if auto_delete?
+      ensure
+        process_cleanup timer
+        executing.make_false
+        @thread = nil
+      end
+
+      def process_cleanup(timer)
+        debug "Processor #{queue} starts cleanup after perform..."
+        timer.shutdown if timer
+        ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
+        processor_finished
+        debug "Processor #{queue} finished cleanup after perform..."
+      end
+
+      def processor_finished
+        fetcher.tell :processor_finished
+      end
+
+      def visibility_extender(queue_visibility, sqs_msg, body)
+        return unless auto_visibility_timeout?
+        interval = [1,queue_visibility/3].max
+        Concurrent::TimerTask.execute execution_interval: interval,
+                                      timeout_interval: interval do
+          begin
+            sqs_msg.visibility_timeout = queue_visibility
+            yield sqs_msg, body if block_given?
+          rescue StandardError => e
+            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      def get_body(sqs_msg)
+        if sqs_msg.is_a? Array
+          sqs_msg.map { |m| parse_body m }
+        else
+          parse_body sqs_msg
+        end
+      end
+
+      def parse_body(sqs_msg)
+        case body_parser
+        when :json then JSON.parse sqs_msg.body
+        when Proc then body_parser.call sqs_msg
+        when :text, nil then sqs_msg.body
+        else body_parser.load sqs_msg.body
+        end
+      rescue => e
+        raise "Error parsing the message body: #{e.message}"
+      end
+    end
+  end
+end