toiler 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fb0e35e85549aef85080de3be1df4680f745d21
-  data.tar.gz: da042af1e5dd0afcefca0bb3818cca82218ecb19
+  metadata.gz: 7ae93426e47a18dc09639561ed7587a92edf718d
+  data.tar.gz: 4642b9ecc2357810f894f816764df5f5ebe78680
 SHA512:
-  metadata.gz: 7d8e30edeabdcb59975e20b5089db2fde3aaeec146cad0e039828f053039e6ee4b10df4a926783bc92e867ec8ccc903968719eb9988dd72f1a32de95eeae088f
-  data.tar.gz: 0ccafdfa4ece45209397679734eef81fe5021ca9223a997548ecb1b6438b8ab7b713fe31d4f0f45b2b75b397eae2fdb5c59bf5a880de10588663db06df5e6e70
+  metadata.gz: 3b047fdb8cce9cee315f7ef58646a0a64724591b6949d6f6a4201d41481a0b0f3a3f76ee8d61ffcaf5b32e5e1de4e93166eff56f474e1e5f4410fc1cf382538b
+  data.tar.gz: 22a1f742e648f17b0c7051f6372f90ca7a619756bcd2bf681344a453456fc11b36c5a06855ef6b06f5a499d2efbea342fd36be29e0c8ea6cd39b0bd72aec403a

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/Gemfile

@@ -1,7 +1,10 @@
 source 'https://rubygems.org'
 
-# gem 'celluloid'
-gem 'celluloid', '~> 0.17.pre15'
-# gem 'celluloid-task-pooledfiber', github: 'celluloid/celluloid-task-pooledfiber', branch: '0.2.0-prerelease'
-# gem 'celluloid-io'
+gem 'concurrent-ruby', '~> 0.9.0'
+gem 'concurrent-ruby-edge', '~> 0.1.0'
 gem 'aws-sdk'
+
+
+group :development do
+  gem 'rubocop'
+end

data/Gemfile.lock

@@ -1,69 +1,38 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.0.42)
-      aws-sdk-resources (= 2.0.42)
-    aws-sdk-core (2.0.42)
-      builder (~> 3.0)
+    ast (2.0.0)
+    astrolabe (1.3.0)
+      parser (>= 2.2.0.pre.3, < 3.0)
+    aws-sdk (2.1.3)
+      aws-sdk-resources (= 2.1.3)
+    aws-sdk-core (2.1.3)
       jmespath (~> 1.0)
-      multi_json (~> 1.0)
-    aws-sdk-resources (2.0.42)
-      aws-sdk-core (= 2.0.42)
-    builder (3.2.2)
-    celluloid (0.17.0.pre15)
-      bundler
-      celluloid-essentials
-      celluloid-extras
-      celluloid-fsm
-      celluloid-pool
-      celluloid-supervision
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    celluloid-essentials (0.20.0.pre17)
-      bundler
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    celluloid-extras (0.1.4)
-      bundler
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    celluloid-fsm (0.9.0.pre13)
-      bundler
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    celluloid-pool (0.11.0.pre1)
-      bundler
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    celluloid-supervision (0.20.0.pre6)
-      bundler
-      dotenv
-      nenv
-      rspec-logsplit (>= 0.1.2)
-      timers (~> 4.0.0)
-    dotenv (2.0.1)
-    hitimes (1.2.2)
+    aws-sdk-resources (2.1.3)
+      aws-sdk-core (= 2.1.3)
+    concurrent-ruby (0.9.0)
+    concurrent-ruby-edge (0.1.0)
+      concurrent-ruby (~> 0.9.0)
     jmespath (1.0.2)
       multi_json (~> 1.0)
-    multi_json (1.11.0)
-    nenv (0.2.0)
-    rspec-logsplit (0.1.3)
-    timers (4.0.1)
-      hitimes
+    multi_json (1.11.2)
+    parser (2.3.0.pre.2)
+      ast (>= 1.1, < 3.0)
+    powerpack (0.1.1)
+    rainbow (2.0.0)
+    rubocop (0.32.1)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.2.5, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.4)
+    ruby-progressbar (1.7.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   aws-sdk
-  celluloid (~> 0.17.pre15)
+  concurrent-ruby (~> 0.9.0)
+  concurrent-ruby-edge (~> 0.1.0)
+  rubocop

data/README.md

@@ -8,6 +8,8 @@ a different approach at loadbalancing and uses long-polling.
 Toiler allows to specify the amount of processors (threads) that should be spawned for each queue.
 Instead of [shoryuken's](https://github.com/phstc/shoryuken) loadbalancing  approach, Toiler delegates this work to the kernel scheduling threads.
 
+Because Toiler uses threads to provide concurrency, **each thread instatiates a new worker**, as it should be expected, so please **use class variables to store shared variables like clients**.
+
 ###Long-Polling
 A Fetcher thread is spawned for each queue.
 Fetchers are resposible for polling SQS and retreiving messages.
@@ -20,6 +22,9 @@ Workers can configure a parser Class or Proc to parse an SQS message body before
 ###Batches
 Toiler allows a Worker to be able to receive a batch of messages instead of a single one.
 
+###Auto Visibility Extension
+Toiler has the ability to automatically extend the visibility timeout of and SQS message to prevent the message from re-entering the queue if processing of such message is taking longer than the queue's visibility timeout.
+
 ##Instalation
 
 Add this line to your application's Gemfile:
@@ -52,7 +57,17 @@ class MyWorker
   # toiler_options auto_visibility_timeout: true
   # toiler_options batch: true
 
+  #Example connection client that should be shared across all instances of MyWorker
+  @@client = ConnectionClient.new
+    
+  def initialize
+    @last_message = nil
+  end
+
   def perform(sqs_msg, body)
+    #Workers are thread safe, yay!
+    #Each worker instance is assured to be processing only one message at a time
+    @last_message = sqs_msg 
     puts body
   end
 end

data/lib/toiler.rb

@@ -1,76 +1,47 @@
 require 'aws-sdk'
-require 'toiler/core_ext'
-require 'toiler/message'
-require 'toiler/queue'
+require 'toiler/utils/environment_loader'
+require 'toiler/utils/logging'
+require 'toiler/utils/argument_parser'
 require 'toiler/worker'
-require 'toiler/environment_loader'
-require 'toiler/logging'
 require 'toiler/cli'
 require 'toiler/version'
 
+# Main module
 module Toiler
-  @worker_registry = {}
   @worker_class_registry = {}
   @options = {
     aws: {}
   }
+  @fetchers = {}
+  @processor_pools = {}
 
-  module_function
+  attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
+  module_function :worker_class_registry, :options, :fetchers, :processor_pools
 
-  def options
-    @options
-  end
+  module_function
 
   def logger
-    Toiler::Logging.logger
-  end
-
-  def worker_class_registry
-    @worker_class_registry
-  end
-
-  def worker_registry
-    @worker_registry
+    Toiler::Utils::Logging.logger
   end
 
   def queues
-    @worker_registry.keys
+    worker_class_registry.keys
   end
 
   def fetcher(queue)
-    Celluloid::Actor["fetcher_#{queue}".to_sym]
+    fetchers["fetcher_#{queue}".to_sym]
   end
 
   def set_fetcher(queue, val)
-    Celluloid::Actor["fetcher_#{queue}".to_sym] = val
+    fetchers["fetcher_#{queue}".to_sym] = val
   end
 
   def processor_pool(queue)
-    Celluloid::Actor["processor_pool_#{queue}".to_sym]
+    processor_pools["processor_pool_#{queue}".to_sym]
   end
 
   def set_processor_pool(queue, val)
-    Celluloid::Actor["processor_pool_#{queue}".to_sym] = val
-  end
-
-  def scheduler(queue)
-    Celluloid::Actor["scheduler_#{queue}".to_sym]
-  end
-
-  def manager
-    Celluloid::Actor[:manager]
-  end
-
-  def set_manager(val)
-    Celluloid::Actor[:manager] = val
-  end
-
-  def timer
-    Celluloid::Actor[:timer]
-  end
-
-  def set_timer(val)
-    Celluloid::Actor[:timer] = val
+    processor_pools["processor_pool_#{queue}".to_sym] = val
   end
 
   def default_options

data/lib/toiler/actor/fetcher.rb

@@ -0,0 +1,89 @@
+require 'toiler/actor/utils/actor_logging'
+require 'toiler/aws/queue'
+
+module Toiler
+  module Actor
+    # Actor polling for messages only when processors are ready, otherwise idle
+    class Fetcher < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      FETCH_LIMIT = 10.freeze
+
+      attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
+                    :scheduled
+
+      def initialize(queue, client)
+        debug "Initializing Fetcher for queue #{queue}..."
+        @queue = Toiler::Aws::Queue.new queue, client
+        @wait = Toiler.options[:wait] || 20
+        @free_processors = Concurrent::AtomicFixnum.new(0)
+        @batch = Toiler.worker_class_registry[queue].batch?
+        @visibility_timeout = @queue.visibility_timeout
+        @scheduled = Concurrent::AtomicBoolean.new
+        debug "Finished initializing Fetcher for queue #{queue}"
+      end
+
+      def default_executor
+        Concurrent.global_fast_executor
+      end
+
+      def on_message(msg)
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Fetcher #{queue.name} raised exception #{e.class}"
+      end
+
+      private
+
+      def batch?
+        @batch
+      end
+
+      def processor_finished
+        debug "Fetcher #{queue.name} received processor finished signal..."
+        free_processors.increment
+        schedule_poll
+      end
+
+      def max_messages
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
+      end
+
+      def poll_future
+        Concurrent.future do
+          queue.receive_messages message_attribute_names: %w(All),
+                                 wait_time_seconds: wait,
+                                 max_number_of_messages: max_messages
+        end
+      end
+
+      def poll_messages
+        poll_future.on_completion! do |_success, msgs, _reason|
+          scheduled.make_false
+          tell [:assign_messages, msgs] unless msgs.nil? || msgs.empty?
+          schedule_poll
+        end
+      end
+
+      def schedule_poll
+        return unless free_processors.value > 0 && scheduled.make_true
+        debug "Fetcher #{queue.name} scheduling polling..."
+        tell :poll_messages
+      end
+
+      def processor_pool
+        @processor_pool ||= Toiler.processor_pool queue.name
+      end
+
+      def assign_messages(messages)
+        messages = [messages] if batch?
+        messages.each do |m|
+          processor_pool.tell [:process, visibility_timeout, m]
+          free_processors.decrement
+        end
+        debug "Fetcher #{queue.name} assigned #{messages.count} messages"
+      end
+    end
+  end
+end

data/lib/toiler/actor/processor.rb

@@ -0,0 +1,106 @@
+require 'json'
+require 'toiler/actor/utils/actor_logging'
+
+module Toiler
+  module Actor
+    # Responsible for processing sqs messages and notifying Fetcher when done
+    class Processor < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      attr_accessor :queue, :worker, :fetcher, :body_parser,
+                    :extend_callback
+
+      def initialize(queue)
+        @queue = queue
+        @worker = Toiler.worker_class_registry[queue].new
+        @fetcher = Toiler.fetcher queue
+        init_options
+        processor_finished
+      end
+
+      def default_executor
+        Concurrent.global_io_executor
+      end
+
+      def on_message(msg)
+        method, *args = msg
+        send(method, *args)
+      rescue StandardError => e
+        error "Processor #{queue} failed processing, reason: #{e.class}"
+        raise e
+      end
+
+      private
+
+      def init_options
+        @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
+        @auto_delete = @worker.class.auto_delete?
+        toiler_options = @worker.class.toiler_options
+        @body_parser = toiler_options[:parser]
+        @extend_callback = toiler_options[:on_visibility_extend]
+      end
+
+      def auto_visibility_timeout?
+        @auto_visibility_timeout
+      end
+
+      def auto_delete?
+        @auto_delete
+      end
+
+      def process(visibility, sqs_msg)
+        debug "Processor #{queue} begins processing..."
+        body = get_body(sqs_msg)
+        timer = visibility_extender visibility, sqs_msg, body, &extend_callback
+
+        debug "Worker #{queue} starts performing..."
+        worker.perform sqs_msg, body
+        debug "Worker #{queue} finishes performing..."
+        sqs_msg.delete if auto_delete?
+      ensure
+        process_cleanup timer
+      end
+
+      def process_cleanup(timer)
+        debug "Processor #{queue} starts cleanup after perform..."
+        timer.shutdown if timer
+        ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
+        processor_finished
+        debug "Processor #{queue} finished cleanup after perform..."
+      end
+
+      def processor_finished
+        fetcher.tell :processor_finished
+      end
+
+      def visibility_extender(queue_visibility, sqs_msg, body)
+        return unless auto_visibility_timeout?
+        interval = queue_visibility - 5
+        Concurrent::TimerTask.execute execution_interval: interval,
+                                      timeout_interval: interval do
+          sqs_msg.visibility_timeout = queue_visibility
+          yield sqs_msg, body if block_given?
+        end
+      end
+
+      def get_body(sqs_msg)
+        if sqs_msg.is_a? Array
+          sqs_msg.map { |m| parse_body m }
+        else
+          parse_body sqs_msg
+        end
+      end
+
+      def parse_body(sqs_msg)
+        case body_parser
+        when :json then JSON.parse sqs_msg.body
+        when Proc then body_parser.call sqs_msg
+        when :text, nil then sqs_msg.body
+        else body_parser.load sqs_msg.body
+        end
+      rescue => e
+        raise "Error parsing the message body: #{e.message}"
+      end
+    end
+  end
+end