toiler 0.1.2 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d01f9405ae62e985f25de5ff0a2f79f67c43c2b2
-  data.tar.gz: 3bdc7a8ced52a7ef39eb5e8af90c84076f3e57df
+  metadata.gz: 3fb0e35e85549aef85080de3be1df4680f745d21
+  data.tar.gz: da042af1e5dd0afcefca0bb3818cca82218ecb19
 SHA512:
-  metadata.gz: 2fc8d6adbff7c7c5ad263e97171a37e9eb70748ca01bbf314ad23401b2e71332eea1da523938ba2248a71ffc507c677a4d1ed9cc1c1dc88cc574d29af6d5fa5d
-  data.tar.gz: bc210eac97b7cf07c5903e65ac9865de9bb8595539e252254b855b9654de7281ebb8a34ae5773e5bc074c7e59db544c73e168b3bc75b293445ab6d7ffb4410de
+  metadata.gz: 7d8e30edeabdcb59975e20b5089db2fde3aaeec146cad0e039828f053039e6ee4b10df4a926783bc92e867ec8ccc903968719eb9988dd72f1a32de95eeae088f
+  data.tar.gz: 0ccafdfa4ece45209397679734eef81fe5021ca9223a997548ecb1b6438b8ab7b713fe31d4f0f45b2b75b397eae2fdb5c59bf5a880de10588663db06df5e6e70

data/.gitmodules

@@ -0,0 +1,4 @@
+[submodule "celluloid-task-pooledfiber"]
+	path = celluloid-task-pooledfiber
+	url = git@github.com:celluloid/celluloid-task-pooledfiber.git
+	branch = 0.2.0-prerelease

data/Gemfile

@@ -1,5 +1,7 @@
 source 'https://rubygems.org'
 
-gem 'celluloid'
-gem 'celluloid-io'
+# gem 'celluloid'
+gem 'celluloid', '~> 0.17.pre15'
+# gem 'celluloid-task-pooledfiber', github: 'celluloid/celluloid-task-pooledfiber', branch: '0.2.0-prerelease'
+# gem 'celluloid-io'
 gem 'aws-sdk'

data/Gemfile.lock

@@ -1,25 +1,63 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.0.41)
-      aws-sdk-resources (= 2.0.41)
-    aws-sdk-core (2.0.41)
+    aws-sdk (2.0.42)
+      aws-sdk-resources (= 2.0.42)
+    aws-sdk-core (2.0.42)
       builder (~> 3.0)
       jmespath (~> 1.0)
       multi_json (~> 1.0)
-    aws-sdk-resources (2.0.41)
-      aws-sdk-core (= 2.0.41)
+    aws-sdk-resources (2.0.42)
+      aws-sdk-core (= 2.0.42)
     builder (3.2.2)
-    celluloid (0.16.0)
+    celluloid (0.17.0.pre15)
+      bundler
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
       timers (~> 4.0.0)
-    celluloid-io (0.16.2)
-      celluloid (>= 0.16.0)
-      nio4r (>= 1.1.0)
+    celluloid-essentials (0.20.0.pre17)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-extras (0.1.4)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-fsm (0.9.0.pre13)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-pool (0.11.0.pre1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    celluloid-supervision (0.20.0.pre6)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (~> 4.0.0)
+    dotenv (2.0.1)
     hitimes (1.2.2)
     jmespath (1.0.2)
       multi_json (~> 1.0)
     multi_json (1.11.0)
-    nio4r (1.1.0)
+    nenv (0.2.0)
+    rspec-logsplit (0.1.3)
     timers (4.0.1)
       hitimes
 
@@ -28,5 +66,4 @@ PLATFORMS
 
 DEPENDENCIES
   aws-sdk
-  celluloid
-  celluloid-io
+  celluloid (~> 0.17.pre15)

data/celluloid-task-pooledfiber/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/celluloid-task-pooledfiber/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/celluloid-task-pooledfiber/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.2.1
+  - rbx-2
+  - jruby

data/celluloid-task-pooledfiber/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gem 'celluloid', github: 'celluloid/celluloid', branch: '0.17.0-prerelease'
+
+gemspec
+
+group :test do
+  gem 'rake'
+  gem 'rspec', '~> 3.2'
+  gem 'rspec-retry'
+end

data/celluloid-task-pooledfiber/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Chris Heald
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/celluloid-task-pooledfiber/README.md

@@ -0,0 +1,37 @@
+# Celluloid::Task::PooledFiber
+
+[![Build Status](https://travis-ci.org/celluloid/celluloid-task-pooledfiber.svg)](https://travis-ci.org/celluloid/celluloid-task-pooledfiber)
+
+Provides a Task implementation which reuses a pool of fibers to run tasks. This improves Celluloid's performance.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'celluloid-task-pooledfiber'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install celluloid-task-pooledfiber
+
+## Usage
+
+In your application, before starting Celluloid:
+
+    Celluloid.task_class = Celluloid::Task::PooledFiber
+
+That's it.
+
+## Contributing
+
+1. Fork it ( https://github.com/cheald/celluloid-task-pooledfiber/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/celluloid-task-pooledfiber/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rspec_opts = ['--color --format documentation']
+end
+task default: :spec

data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec

@@ -0,0 +1,18 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'celluloid-task-pooledfiber'
+  spec.version       = '0.2.0'
+  spec.authors       = ['Chris Heald']
+  spec.email         = ['cheald@gmail.com']
+
+  spec.summary       = 'An alternate Task implementation for Celluloid which improves performance by reusing a pool of fibers to run tasks'
+  spec.description   = 'An alternate Task implementation for Celluloid which improves performance by reusing a pool of fibers to run tasks'
+  spec.homepage      = 'http://github.com/cheald/celluloid-task-pooledfiber'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|examples|spec|features)/}) }
+  spec.require_paths = ['lib']
+end

data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb

@@ -0,0 +1,26 @@
+require 'celluloid' unless defined? Celluloid
+require 'thread'
+
+require 'celluloid/util/fiber_pool'
+
+module Celluloid
+  class Task
+    # Tasks with a pooled Fiber backend
+    class PooledFiber < Fibered
+      def self.fiber_pool
+        @fiber_pool ||= Util::FiberPool.new
+      end
+
+      def create(&block)
+        queue = Thread.current[:celluloid_queue]
+        actor_system = Thread.current[:celluloid_actor_system]
+        @fiber = PooledFiber.fiber_pool.acquire do
+          Thread.current[:celluloid_role] = :actor
+          Thread.current[:celluloid_queue] = queue
+          Thread.current[:celluloid_actor_system] = actor_system
+          block.call
+        end
+      end
+    end
+  end
+end

data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb

@@ -0,0 +1,95 @@
+module Celluloid
+  module Util
+    class FiberPool
+      attr_accessor :stats
+
+      def initialize(trim_size = 64)
+        @trim_size = trim_size
+        @pool = {}
+        @stats = { created: 0, acquired: 0, trimmed: 0, sweep_counter: 0, terminated: 0, terminated_threads: 0, sweeps: 0 }
+        @mutex = Mutex.new
+      end
+
+      def acquire(&block)
+        trim
+        sweep
+        @stats[:acquired] += 1
+        fiber = fiber_pool.shift || create_fiber
+        fiber.resume(block)
+        fiber
+      end
+
+      private
+
+      def create_fiber
+        pool = fiber_pool
+        @stats[:created] += 1
+        Fiber.new do |blk|
+          loop do
+            # At this point, we have a pooled fiber ready for Celluloid to call #resume on.
+            Fiber.yield
+
+            # Once Celluloid resumes the fiber, we need to execute the block the task was
+            # created with. This may call suspend/resume; that's fine.
+            blk.call
+
+            # Once Celluloid completes the task, we release this Fiber back to the pool
+            pool << Fiber.current
+
+            # ...and yield for the next #acquire call.
+            blk = Fiber.yield
+            break if blk == :terminate
+          end
+        end
+      end
+
+      # If the fiber pool has grown too much, shift off and discard
+      # extra fibers so they may be GC'd. This isn't ideal, but it
+      # should guard against runaway resource consumption.
+      def trim
+        pool = fiber_pool
+        while pool.length > @trim_size
+          @stats[:trimmed] += 1
+          pool.shift.resume(:terminate)
+        end
+      end
+
+      def sweep
+        @mutex.synchronize do
+          @stats[:sweep_counter] += 1
+          if @stats[:sweep_counter] > 10_000
+            alive = []
+            Thread.list.each do |thread|
+              alive << thread.object_id if thread.alive? && @pool.key?(thread.object_id)
+            end
+
+            (@pool.keys - alive).each do |thread_id|
+              @pool[thread_id].each do |_fiber|
+                @stats[:terminated] += 1
+                # We can't resume the fiber here because we might resume cross-thread
+                # TODO: How do we deal with alive fibers in a dead thread?
+                # fiber.resume(:terminate)
+              end
+              @stats[:terminated_threads] += 1
+              @pool.delete thread_id
+            end
+            @stats[:sweep_counter] = 0
+            @stats[:sweeps] += 1
+          end
+        end
+      end
+
+      # Fiber pool for this thread. Fibers can't cross threads, so we have to maintain a
+      # pool per thread.
+      #
+      # We keep our pool in an instance variable rather than in thread locals so that we
+      # can sweep out old Fibers from dead threads. This keeps live Fiber instances in
+      # a thread local from keeping the thread from being GC'd.
+      def fiber_pool
+        @mutex.synchronize do
+          @pool[Thread.current.object_id] ||= []
+        end
+      end
+    end
+  end
+end

data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+RSpec.describe Celluloid::Task::PooledFiber, actor_system: :within do
+  it_behaves_like 'a Celluloid Task'
+end

data/celluloid-task-pooledfiber/spec/spec_helper.rb

@@ -0,0 +1,60 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'celluloid/test'
+require 'celluloid/rspec'
+
+require 'celluloid/tasks/pooled_fiber'
+
+$CELLULOID_DEBUG = true
+
+require 'celluloid/probe'
+
+Dir['./spec/support/*.rb'].map { |f| require f }
+
+RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
+  config.around do |ex|
+    Celluloid.actor_system = nil
+    Thread.list.each do |thread|
+      next if thread == Thread.current
+      if defined?(JRUBY_VERSION)
+        # Avoid disrupting jRuby's "fiber" threads.
+        next if /Fiber/ =~ thread.to_java.getNativeThread.get_name
+      end
+      thread.kill
+    end
+
+    ex.run
+  end
+
+  config.around actor_system: :global do |ex|
+    Celluloid.boot
+    ex.run
+    Celluloid.shutdown
+  end
+
+  config.around actor_system: :within do |ex|
+    Celluloid::ActorSystem.new.within do
+      ex.run
+    end
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_doubled_constant_names = true
+    mocks.verify_partial_doubles = true
+  end
+
+  config.around(:each) do |example|
+    example.run
+  end
+
+  # Must be *after* the around hook above
+  require 'rspec/retry'
+  config.verbose_retry = true
+  config.default_sleep_interval = 3
+end

data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb

@@ -0,0 +1,49 @@
+class MockActor
+  attr_reader :tasks
+
+  def initialize
+    @tasks = []
+  end
+
+  def setup_thread
+  end
+end
+
+RSpec.shared_examples 'a Celluloid Task' do
+  let(:task_type)     { :foobar }
+  let(:suspend_state) { :doing_something }
+  let(:actor)         { MockActor.new }
+
+  subject { Celluloid.task_class.new(task_type, {}) { Celluloid::Task.suspend(suspend_state) } }
+
+  before :each do
+    Thread.current[:celluloid_actor_system] = Celluloid.actor_system
+    Thread.current[:celluloid_actor] = actor
+  end
+
+  after :each do
+    Thread.current[:celluloid_actor] = nil
+    Thread.current[:celluloid_actor_system] = nil
+  end
+
+  it 'begins with status :new' do
+    expect(subject.status).to be :new
+  end
+
+  it 'resumes' do
+    expect(subject).to be_running
+    subject.resume
+    expect(subject.status).to eq(suspend_state)
+    subject.resume
+    expect(subject).not_to be_running
+  end
+
+  it 'raises exceptions outside' do
+    task = Celluloid.task_class.new(task_type, {}) do
+      fail 'failure'
+    end
+    expect do
+      task.resume
+    end.to raise_exception('failure')
+  end
+end

data/lib/toiler/cli.rb

@@ -1,4 +1,5 @@
 require 'singleton'
+require 'timeout'
 require 'optparse'
 require 'toiler'
 
@@ -30,6 +31,7 @@ module Toiler
       load_celluloid
 
       begin
+        require 'toiler/supervisor'
         @supervisor = Supervisor.new
 
         while (readable_io = IO.select([self_read]))
@@ -37,8 +39,14 @@ module Toiler
           handle_signal(signal)
         end
       rescue Interrupt
-        @supervisor.stop
-        exit 0
+        puts 'Received interrupt, terminating actors...'
+        begin
+          Timeout.timeout(20) do
+            @supervisor.stop
+          end
+        ensure
+          exit 0
+        end
       end
     end
 
@@ -54,9 +62,10 @@ module Toiler
       # Celluloid can't be loaded until after we've daemonized
       # because it spins up threads and creates locks which get
       # into a very bad state if forked.
-      require 'celluloid/autostart'
+      require 'celluloid/current'
+      require 'celluloid/task/pooled_fiber'
+      Celluloid.task_class = Celluloid::Task::PooledFiber
       Celluloid.logger = (Toiler.options[:verbose] ? Toiler.logger : nil)
-      require 'toiler/supervisor'
     end
 
     def daemonize
@@ -74,7 +83,7 @@ module Toiler
       files_to_reopen.each do |file|
         begin
           file.reopen file.path, 'a+'
-          #file.sync = true
+          # file.sync = true
         rescue ::Exception
         end
       end
@@ -83,7 +92,7 @@ module Toiler
         File.open(Toiler.options[:logfile], 'ab') do |f|
           io.reopen(f)
         end
-        #io.sync = true
+        # io.sync = true
       end
       $stdin.reopen('/dev/null')
     end

data/lib/toiler/fetcher.rb

@@ -1,16 +1,17 @@
 module Toiler
   class Fetcher
     include Celluloid
-    include Celluloid::Logger
+    include Celluloid::Internals::Logger
 
     FETCH_LIMIT = 10.freeze
 
-    attr_accessor :queue, :wait, :batch
+    attr_accessor :queue, :wait, :batch, :condition
 
     finalizer :shutdown
 
-    def initialize(queue, client = nil)
+    def initialize(queue, client)
       debug "Initializing Fetcher for queue #{queue}..."
+      @condition = Celluloid::Condition.new
       @queue = Queue.new queue, client
       @wait = Toiler.options[:wait] || 20
       @batch = Toiler.worker_class_registry[queue].batch?
@@ -23,6 +24,15 @@ module Toiler
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
+    def processor_finished
+      @condition.broadcast
+    end
+
+    def wait_for_available_processors
+      manager = Toiler.manager
+      @condition.wait if manager.dead? || manager.free_processors(queue) == 0
+    end
+
     def poll_messages
       # AWS limits the batch size by 10
       options = {
@@ -31,14 +41,15 @@ module Toiler
       }
 
       loop do
-        count = Toiler.manager.free_processors queue.name
+        while (count = Toiler.manager.free_processors(queue.name)) == 0
+          wait_for_available_processors
+        end
         options[:max_number_of_messages] = (batch || count > FETCH_LIMIT) ? FETCH_LIMIT : count
         debug "Fetcher #{queue.name} retreiving messages with options: #{options.inspect}..."
         msgs = queue.receive_messages options
         debug "Fetcher #{queue.name} retreived #{msgs.count} messages..."
         next if msgs.empty?
         Toiler.manager.assign_messages queue.name, msgs
-        Toiler.manager.wait_for_available_processors queue.name
       end
     end
   end

data/lib/toiler/manager.rb

@@ -4,91 +4,51 @@ require 'toiler/processor'
 module Toiler
   class Manager
     include Celluloid
-    include Celluloid::Logger
-
-    attr_accessor :queues, :client
+    include Celluloid::Internals::Logger
 
     finalizer :shutdown
 
     def initialize
-      Toiler.set_manager current_actor
       async.init
     end
 
     def init
       debug 'Initializing manager...'
-      @queues = Toiler.worker_class_registry
-      @client = ::Aws::SQS::Client.new
       init_workers
-      init_conditions
-      pool_processors
-      supervise_fetchers
+      awake_fetchers
       debug 'Finished initializing manager...'
     end
 
+    def awake_fetchers
+      queues.each do |q, _klass|
+        fetcher = Toiler.fetcher(q)
+        fetcher.processor_finished if fetcher && fetcher.alive? && free_processors(q) > 0
+      end
+    end
+
+    def queues
+      Toiler.worker_class_registry
+    end
+
     def shutdown
       debug 'Manager shutting down...'
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
-    def stop
-      debug 'Manager stopping...'
-      terminate_fetchers
-      terminate_processors
-    end
-
     def processor_finished(queue)
-      @conditions[queue].broadcast
+      fetcher = Toiler.fetcher(queue)
+      fetcher.processor_finished if fetcher && fetcher.alive?
     end
 
     def init_workers
-      Toiler.worker_class_registry.each do |q, klass|
-        Toiler.worker_registry[q] = klass.new
-      end
-    end
-
-    def supervise_fetchers
-      queues.each do |queue, _klass|
-        Toiler.set_fetcher queue, Fetcher.supervise(queue, client).actors.first
-      end
-    end
-
-    def pool_processors
       queues.each do |q, klass|
-        count = klass.concurrency
-        processor = if count > 1
-                      Processor.pool args: [q], size: count
-                    else
-                      Processor.supervise(q).actors.first
-                    end
-        Toiler.set_processor_pool q, processor
-      end
-    end
-
-    def terminate_fetchers
-      queues.each do |queue, _klass|
-        fetcher = Toiler.fetcher(queue)
-        fetcher.terminate if fetcher && fetcher.alive?
-      end
-    end
-
-    def terminate_processors
-      queues.each do |queue, _klass|
-        processor_pool = Toiler.processor_pool(queue)
-        processor_pool.terminate if processor_pool && processor_pool.alive?
-      end
-    end
-
-    def init_conditions
-      @conditions = {}
-      queues.each do |queue, _klass|
-        @conditions[queue] = Celluloid::Condition.new
+        Toiler.worker_registry[q] = klass.new
       end
     end
 
     def free_processors(queue)
-      return 1 unless Toiler.processor_pool(queue).respond_to? :idle_size
-      Toiler.processor_pool(queue).idle_size
+      pool = Toiler.processor_pool(queue)
+      pool && pool.alive? ? pool.idle_size : 0
     end
 
     def assign_messages(queue, messages)
@@ -104,12 +64,8 @@ module Toiler
       debug "Manager finished assigning #{messages.count} for queue #{queue}"
     end
 
-    def wait_for_available_processors(queue)
-      @conditions[queue].wait if free_processors(queue) == 0
-    end
-
     def batch?(queue)
-      Toiler.worker_class_registry[queue].batch?
+      queues[queue].batch?
     end
   end
 end

data/lib/toiler/processor.rb

@@ -4,51 +4,48 @@ require 'toiler/scheduler'
 module Toiler
   class Processor
     include Celluloid
-    include Celluloid::Logger
+    include Celluloid::Internals::Logger
 
-    attr_accessor :queue, :scheduler
+    attr_accessor :queue
 
     finalizer :shutdown
 
     def initialize(queue)
       debug "Initializing Processor for queue #{queue}"
       @queue = queue
-      @scheduler = Scheduler.supervise.actors.first
       processor_finished
       debug "Finished initializing Processor for queue #{queue}"
     end
 
     def shutdown
       debug "Processor for queue #{queue} shutting down..."
-      scheduler.terminate if scheduler && scheduler.alive?
+      ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
       instance_variables.each { |iv| remove_instance_variable iv }
     end
 
     def process(queue, sqs_msg)
       debug "Processor #{queue} begins processing..."
-      exclusive do
-        worker = Toiler.worker_registry[queue]
-        timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
+      worker = Toiler.worker_registry[queue]
+      timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
 
-        begin
-          body = get_body(worker.class, sqs_msg)
-          worker.perform(sqs_msg, body)
-          sqs_msg.delete if worker.class.auto_delete?
-        ensure
-          timer.cancel if timer
-          ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
-        end
-      end
+      body = get_body(worker.class, sqs_msg)
+      worker.perform(sqs_msg, body)
+      sqs_msg.delete if worker.class.auto_delete?
+    rescue StandardError => e
+      error "Processor #{queue} faild processing msg: #{e.message}\n#{e.backtrace.join("\n")}"
+    ensure
+      timer.cancel if timer
+      ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
       processor_finished
       debug "Processor #{queue} finishes processing..."
     end
 
+    private
+
     def processor_finished
-      Toiler.manager.processor_finished queue
+      Toiler.manager.async.processor_finished queue
     end
 
-    private
-
     def auto_visibility_timeout(queue, sqs_msg, worker_class)
       return unless worker_class.auto_visibility_timeout?
       queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
@@ -57,7 +54,7 @@ module Toiler
         msg.visibility_timeout = visibility_timeout
       end
 
-      scheduler.custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
+      Toiler.scheduler(queue).custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
     end
 
     def get_body(worker_class, sqs_msg)

data/lib/toiler/scheduler.rb

@@ -1,7 +1,7 @@
 module Toiler
   class Scheduler
     include Celluloid
-    include Celluloid::Logger
+    include Celluloid::Internals::Logger
 
     execute_block_on_receiver :custom_every
 

data/lib/toiler/supervisor.rb

@@ -1,24 +1,66 @@
 require 'toiler/manager'
 
 module Toiler
-  class Supervisor < Celluloid::SupervisionGroup
-    include Celluloid
-
-    finalizer :shutdown
+  class Supervisor
+    attr_accessor :client, :config
 
     def initialize
-      @manager = Manager.new
+      @client = ::Aws::SQS::Client.new
+      @config = Celluloid::Supervision::Configuration.new
+      define_manager
+      define_schedulers
+      define_processors
+      define_fetchers
+      @config.deploy
+    end
+
+    def queues
+      Toiler.worker_class_registry
+    end
+
+    def define_manager
+      @config.define type: Manager, as: :manager
+    end
+
+    def define_fetchers
+      queues.each do |queue, _klass|
+        @config.define type: Fetcher, as: "fetcher_#{queue}".to_sym, args: [queue, client]
+      end
+    end
+
+    def define_schedulers
+      queues.each do |queue, _klass|
+        @config.define type: Scheduler, as: "scheduler_#{queue}".to_sym, args: []
+      end
+    end
+
+    def define_processors
+      queues.each do |q, klass|
+        @config.define type: Celluloid::Supervision::Container::Pool,
+                       as: "processor_pool_#{q}".to_sym,
+                       args: [actors: Processor, size: klass.concurrency, args: [q]]
+      end
     end
 
     def stop
-      return unless @manager.alive?
-      @manager.stop
-      @manager.terminate
+      terminate_fetchers
+      terminate_processors
+      Toiler.manager.terminate if Toiler.manager && Toiler.manager.alive?
+      @config.shutdown
+    end
+
+    def terminate_fetchers
+      queues.each do |queue, _klass|
+        fetcher = Toiler.fetcher(queue)
+        fetcher.terminate if fetcher && fetcher.alive?
+      end
     end
 
-    def shutdown
-      @manager.terminate if @manager.alive?
-      instance_variables.each { |iv| remove_instance_variable iv }
+    def terminate_processors
+      queues.each do |queue, _klass|
+        processor_pool = Toiler.processor_pool(queue)
+        processor_pool.terminate if processor_pool && processor_pool.alive?
+      end
     end
   end
 end

data/lib/toiler/version.rb

@@ -1,3 +1,3 @@
 module Toiler
-  VERSION = '0.1.2'
+  VERSION = '0.1.5'
 end

data/lib/toiler.rb

@@ -53,6 +53,10 @@ module Toiler
     Celluloid::Actor["processor_pool_#{queue}".to_sym] = val
   end
 
+  def scheduler(queue)
+    Celluloid::Actor["scheduler_#{queue}".to_sym]
+  end
+
   def manager
     Celluloid::Actor[:manager]
   end

data/toiler.gemspec

@@ -13,9 +13,10 @@ Gem::Specification.new do |spec|
   spec.license       = 'LGPLv3'
 
   spec.files         = `git ls-files -z`.split("\x0")
+  spec.files += `cd celluloid-task-pooledfiber && git ls-files -z`.split("\x0").collect { |f| "celluloid-task-pooledfiber/#{f}" }
   spec.executables << 'toiler'
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
+  spec.require_paths = ['lib', 'celluloid-task-pooledfiber/lib']
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
@@ -25,5 +26,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'dotenv'
 
   spec.add_dependency 'aws-sdk', '~> 2.0.21'
-  spec.add_dependency 'celluloid', '~> 0.16.0'
+  spec.add_dependency 'celluloid', '~> 0.17.pre15'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.5
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-07 00:00:00.000000000 Z
+date: 2015-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.pre15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.pre15
 description: Toiler is a super efficient AWS SQS thread based message processor
 email:
 - sebas.schep@hotmail.com
@@ -131,12 +131,26 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitmodules"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/toiler
+- celluloid-task-pooledfiber/.gitignore
+- celluloid-task-pooledfiber/.rspec
+- celluloid-task-pooledfiber/.travis.yml
+- celluloid-task-pooledfiber/Gemfile
+- celluloid-task-pooledfiber/LICENSE.txt
+- celluloid-task-pooledfiber/README.md
+- celluloid-task-pooledfiber/Rakefile
+- celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec
+- celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb
+- celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb
+- celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb
+- celluloid-task-pooledfiber/spec/spec_helper.rb
+- celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb
 - lib/toiler.rb
 - lib/toiler/cli.rb
 - lib/toiler/core_ext.rb
@@ -160,6 +174,7 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
+- celluloid-task-pooledfiber/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="