dat-worker-pool 0.4.0 → 0.5.0

data/dat-worker-pool.gemspec

@@ -20,6 +20,6 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency("SystemTimer", ["~> 1.2"])
 
-  gem.add_development_dependency("assert", ["~> 2.12"])
+  gem.add_development_dependency("assert", ["~> 2.14"])
 
 end

data/lib/dat-worker-pool.rb

@@ -8,10 +8,9 @@ require 'dat-worker-pool/worker'
 
 class DatWorkerPool
 
-  TimeoutError = Class.new(RuntimeError)
-
   attr_reader :logger, :spawned
   attr_reader :queue
+  attr_reader :on_worker_error_callbacks
   attr_reader :on_worker_start_callbacks, :on_worker_shutdown_callbacks
   attr_reader :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
   attr_reader :before_work_callbacks, :after_work_callbacks
@@ -30,20 +29,53 @@ class DatWorkerPool
     @workers = []
     @spawned = 0
 
+    @on_worker_error_callbacks    = []
     @on_worker_start_callbacks    = []
     @on_worker_shutdown_callbacks = [proc{ |worker| despawn_worker(worker) }]
     @on_worker_sleep_callbacks    = [proc{ @workers_waiting.increment }]
     @on_worker_wakeup_callbacks   = [proc{ @workers_waiting.decrement }]
-    @before_work_callbacks = []
-    @after_work_callbacks  = []
+    @before_work_callbacks        = []
+    @after_work_callbacks         = []
+
+    @started = false
+  end
+
+  def start
+    @started = true
+    @queue.start
+    @min_workers.times{ spawn_worker }
+  end
 
+  # * All work on the queue is left on the queue. It's up to the controlling
+  #   system to decide how it should handle this.
+  def shutdown(timeout = nil)
     @started = false
+    begin
+      OptionalTimeout.new(timeout){ graceful_shutdown }
+    rescue TimeoutError
+      force_shutdown(timeout, caller)
+    end
+  end
+
+  # * Always check if all workers are busy before pushing the work because
+  #   `@queue.push` can wakeup a worker. If you check after, you can see all
+  #   workers are busy because one just wokeup to handle what was just pushed.
+  #   This would cause it to spawn a worker when one isn't needed.
+  def add_work(work_item)
+    return if work_item.nil?
+    new_worker_needed = self.all_spawned_workers_are_busy?
+    @queue.push work_item
+    spawn_worker if @started && new_worker_needed && !reached_max_workers?
   end
 
   def work_items
     @queue.work_items
   end
 
+  def queue_empty?
+    @queue.empty?
+  end
+
   def waiting
     @workers_waiting.count
   end
@@ -60,107 +92,89 @@ class DatWorkerPool
     @mutex.synchronize{ @spawned >= @max_workers }
   end
 
-  def queue_empty?
-    @queue.empty?
-  end
-
-  # Check if all workers are busy before adding the work. When the work is
-  # added, a worker will stop waiting (if it was idle). Because of that, we
-  # can't reliably check if all workers are busy. We might think all workers are
-  # busy because we just woke up a sleeping worker to process this work. Then we
-  # would spawn a worker to do nothing.
-  def add_work(work_item)
-    return if work_item.nil?
-    new_worker_needed = all_spawned_workers_are_busy?
-    @queue.push work_item
-    self.spawn_worker if @started && new_worker_needed && !reached_max_workers?
-  end
-
-  def start
-    @started = true
-    @queue.start
-    @min_workers.times{ self.spawn_worker }
-  end
-
-  # Shutdown each worker and then the queue. Shutting down the queue will
-  # signal any workers waiting on it to wake up, so they can start shutting
-  # down. If a worker is processing work, then it will be joined and allowed to
-  # finish.
-  # **NOTE** Any work that is left on the queue isn't processed. The controlling
-  # application for the worker pool should gracefully handle these items.
-  def shutdown(timeout = nil)
-    @started = false
-    begin
-      proc = OptionalTimeoutProc.new(timeout, true) do
-        # Workers need to be shutdown before the queue. This marks a flag that
-        # tells the workers to exit out of their loop once they wakeup. The
-        # queue shutdown signals the workers to wakeup, so the flag needs to be
-        # set before they wakeup.
-        @workers.each(&:shutdown)
-        @queue.shutdown
-
-        # use this pattern instead of `each` -- we don't want to call `join` on
-        # every worker (especially if they are shutting down on their own), we
-        # just want to make sure that any who haven't had a chance to finish
-        # get to (this is safe, otherwise you might get a dead thread in the
-        # `each`).
-        @workers.first.join until @workers.empty?
-      end
-      proc.call
-    rescue TimeoutError => exception
-      exception.message.replace "Timed out shutting down the worker pool"
-      @debug ? raise(exception) : self.logger.error(exception.message)
-    end
-  end
-
   def on_queue_pop_callbacks;  @queue.on_pop_callbacks;  end
   def on_queue_push_callbacks; @queue.on_push_callbacks; end
 
-  def on_queue_pop(&block);  @queue.on_pop_callbacks << block;  end
+  def on_queue_pop(&block);  @queue.on_pop_callbacks  << block; end
   def on_queue_push(&block); @queue.on_push_callbacks << block; end
 
-  def on_worker_start(&block);    @on_worker_start_callbacks << block;    end
+  def on_worker_error(&block);    @on_worker_error_callbacks    << block; end
+  def on_worker_start(&block);    @on_worker_start_callbacks    << block; end
   def on_worker_shutdown(&block); @on_worker_shutdown_callbacks << block; end
-  def on_worker_sleep(&block);    @on_worker_sleep_callbacks << block;    end
-  def on_worker_wakeup(&block);   @on_worker_wakeup_callbacks << block;   end
+  def on_worker_sleep(&block);    @on_worker_sleep_callbacks    << block; end
+  def on_worker_wakeup(&block);   @on_worker_wakeup_callbacks   << block; end
 
   def before_work(&block); @before_work_callbacks << block; end
-  def after_work(&block);  @after_work_callbacks << block;  end
+  def after_work(&block);  @after_work_callbacks  << block; end
+
+  private
+
+  def do_work(work_item)
+    @do_work_proc.call(work_item)
+  end
 
-  protected
+  # * Always shutdown workers before the queue. `@queue.shutdown` wakes up the
+  #   workers. If you haven't told them to shutdown before they wakeup then they
+  #   won't start their shutdown when they are woken up.
+  # * Use `@workers.first.join until @workers.empty?` instead of `each` to join
+  #   all the workers. While we are joining a worker a separate worker can
+  #   shutdown and remove itself from the `@workers` array.
+  def graceful_shutdown
+    @workers.each(&:shutdown)
+    @queue.shutdown
+    @workers.first.join until @workers.empty?
+  end
+
+  # * Use `@workers.first until @workers.empty?` pattern instead of `each` to
+  #   raise and join all the workers. While we are raising and joining a worker
+  #   a separate worker can shutdown and remove itself from the `@workers`
+  #   array.
+  # * `rescue false` when joining the workers. Ruby will raise any exceptions
+  #   that aren't handled by a thread when its joined. This ensures if the hard
+  #   shutdown is raised and not rescued (for example, in the workers ensure),
+  #   then it won't cause the forced shutdown to end prematurely.
+  def force_shutdown(timeout, backtrace)
+    error = ShutdownError.new "Timed out shutting down the worker pool " \
+                              "(#{timeout} seconds)."
+    error.set_backtrace(backtrace)
+    until @workers.empty?
+      worker = @workers.first
+      worker.raise(error)
+      worker.join rescue false
+    end
+    raise error if @debug
+  end
 
   def spawn_worker
-    @mutex.synchronize do
-      Worker.new(@queue).tap do |w|
-        w.on_work = proc{ |worker, work_item| do_work(work_item) }
-        w.on_start_callbacks    = @on_worker_start_callbacks
-        w.on_shutdown_callbacks = @on_worker_shutdown_callbacks
-        w.on_sleep_callbacks    = @on_worker_sleep_callbacks
-        w.on_wakeup_callbacks   = @on_worker_wakeup_callbacks
-        w.before_work_callbacks = @before_work_callbacks
-        w.after_work_callbacks  = @after_work_callbacks
-
-        @workers << w
-        @spawned += 1
-
-        w.start
-      end
+    @mutex.synchronize{ spawn_worker! }
+  end
+
+  def spawn_worker!
+    Worker.new(@queue).tap do |w|
+      w.on_work = proc{ |worker, work_item| do_work(work_item) }
+
+      w.on_error_callbacks    = @on_worker_error_callbacks
+      w.on_start_callbacks    = @on_worker_start_callbacks
+      w.on_shutdown_callbacks = @on_worker_shutdown_callbacks
+      w.on_sleep_callbacks    = @on_worker_sleep_callbacks
+      w.on_wakeup_callbacks   = @on_worker_wakeup_callbacks
+      w.before_work_callbacks = @before_work_callbacks
+      w.after_work_callbacks  = @after_work_callbacks
+
+      @workers << w
+      @spawned += 1
+
+      w.start
     end
   end
 
   def despawn_worker(worker)
-    @mutex.synchronize do
-      @spawned -= 1
-      @workers.delete worker
-    end
+    @mutex.synchronize{ despawn_worker!(worker) }
   end
 
-  def do_work(work_item)
-    @do_work_proc.call(work_item)
-  rescue Exception => exception
-    self.logger.error "Exception raised while doing work!"
-    self.logger.error "#{exception.class}: #{exception.message}"
-    self.logger.error exception.backtrace.join("\n")
+  def despawn_worker!(worker)
+    @spawned -= 1
+    @workers.delete worker
   end
 
   class WorkersWaiting
@@ -180,22 +194,12 @@ class DatWorkerPool
     end
   end
 
-  class OptionalTimeoutProc
-    def initialize(timeout, reraise = false, &proc)
-      @timeout = timeout
-      @reraise = reraise
-      @proc    = proc
-    end
-
-    def call
-      if @timeout
-        begin
-          SystemTimer.timeout(@timeout, TimeoutError, &@proc)
-        rescue TimeoutError
-          raise if @reraise
-        end
+  module OptionalTimeout
+    def self.new(seconds, &block)
+      if seconds
+        SystemTimer.timeout(seconds, TimeoutError, &block)
       else
-        @proc.call
+        block.call
       end
     end
   end
@@ -206,4 +210,11 @@ class DatWorkerPool
     end
   end
 
+  TimeoutError = Class.new(RuntimeError)
+
+  # * This error should never be "swallowed". If it is caught be sure to
+  #   re-raise it so the workers shutdown. Otherwise workers will get killed
+  #   (`Thread#kill`) by ruby which causes lots of issues.
+  ShutdownError = Class.new(Interrupt)
+
 end

data/lib/dat-worker-pool/queue.rb

@@ -7,21 +7,28 @@ class DatWorkerPool
     attr_accessor :on_push_callbacks, :on_pop_callbacks
 
     def initialize
-      @work_items = []
-      @shutdown = false
-      @mutex = Mutex.new
+      @work_items         = []
+      @shutdown           = false
+      @mutex              = Mutex.new
       @condition_variable = ConditionVariable.new
 
       @on_pop_callbacks  = []
       @on_push_callbacks = []
     end
 
-    def work_items
-      @mutex.synchronize{ @work_items }
+    def start
+      @shutdown = false
+    end
+
+    # * Wakes up any threads (`@condition_variable.broadcast`) who are sleeping
+    #   because of `pop`.
+    def shutdown
+      @shutdown = true
+      @mutex.synchronize{ @condition_variable.broadcast }
     end
 
-    # Add the work_item and wake up the first worker (the `signal`) that's
-    # waiting (because of `wait_for_work_item`)
+    # * Add the work and wake up the first thread waiting from calling `pop`
+    #  (`@condition_variable.signal`).
     def push(work_item)
       raise "Unable to add work while shutting down" if @shutdown
       @mutex.synchronize do
@@ -31,6 +38,8 @@ class DatWorkerPool
       @on_push_callbacks.each(&:call)
     end
 
+    # * Sleeps the current thread (`@condition_variable.wait(@mutex)`) until it
+    #   is signaled via `push` or `shutdown`.
     def pop
       return if @shutdown
       item = @mutex.synchronize do
@@ -41,18 +50,12 @@ class DatWorkerPool
       item
     end
 
-    def empty?
-      @mutex.synchronize{ @work_items.empty? }
-    end
-
-    def start
-      @shutdown = false
+    def work_items
+      @mutex.synchronize{ @work_items }
     end
 
-    # wake up any workers who are idle (because of `wait_for_work_item`)
-    def shutdown
-      @shutdown = true
-      @mutex.synchronize{ @condition_variable.broadcast }
+    def empty?
+      @mutex.synchronize{ @work_items.empty? }
     end
 
     def shutdown?

data/lib/dat-worker-pool/version.rb

@@ -1,3 +1,3 @@
 class DatWorkerPool
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/dat-worker-pool/worker.rb

@@ -1,10 +1,11 @@
 require 'thread'
+require 'dat-worker-pool'
 
 class DatWorkerPool
 
   class Worker
 
-    attr_accessor :on_work
+    attr_accessor :on_work, :on_error_callbacks
     attr_accessor :on_start_callbacks, :on_shutdown_callbacks
     attr_accessor :on_sleep_callbacks, :on_wakeup_callbacks
     attr_accessor :before_work_callbacks, :after_work_callbacks
@@ -12,6 +13,7 @@ class DatWorkerPool
     def initialize(queue)
       @queue = queue
       @on_work = proc{ |worker, work_item| }
+      @on_error_callbacks    = []
       @on_start_callbacks    = []
       @on_shutdown_callbacks = []
       @on_sleep_callbacks    = []
@@ -27,40 +29,65 @@ class DatWorkerPool
       @thread ||= Thread.new{ work_loop }
     end
 
-    def running?
-      @thread && @thread.alive?
-    end
-
     def shutdown
       @shutdown = true
     end
 
+    def running?
+      !!(@thread && @thread.alive?)
+    end
+
     def join(*args)
       @thread.join(*args) if running?
     end
 
-    protected
+    def raise(*args)
+      @thread.raise(*args) if running?
+    end
+
+    private
 
+    # * Rescue `ShutdownError` but don't do anything with it. We want to handle
+    #   the error but we just want it to cause the worker to exit its work loop.
+    #   If the `ShutdownError` isn't rescued, it will be raised when the worker
+    #   is joined.
     def work_loop
       @on_start_callbacks.each{ |p| p.call(self) }
       loop do
-        @on_sleep_callbacks.each{ |p| p.call(self) }
-        work_item = @queue.pop
-        @on_wakeup_callbacks.each{ |p| p.call(self) }
         break if @shutdown
-        do_work(work_item) if work_item
+        fetch_and_do_work
       end
+    rescue ShutdownError
     ensure
       @on_shutdown_callbacks.each{ |p| p.call(self) }
       @thread = nil
     end
 
+    # * Rescue `ShutdownError` but re-raise it after calling the error
+    #   callbacks. This ensures it causes the work loop to exit (see
+    #   `work_loop`).
+    def fetch_and_do_work
+      @on_sleep_callbacks.each{ |p| p.call(self) }
+      work_item = @queue.pop
+      @on_wakeup_callbacks.each{ |p| p.call(self) }
+      do_work(work_item) if work_item
+    rescue ShutdownError => exception
+      handle_exception(exception, work_item)
+      raise exception
+    rescue StandardError => exception
+      handle_exception(exception, work_item)
+    end
+
     def do_work(work_item)
       @before_work_callbacks.each{ |p| p.call(self, work_item) }
       @on_work.call(self, work_item)
       @after_work_callbacks.each{ |p| p.call(self, work_item) }
     end
 
+    def handle_exception(exception, work_item = nil)
+      @on_error_callbacks.each{ |p| p.call(self, exception, work_item) }
+    end
+
   end
 
 end

data/lib/dat-worker-pool/worker_pool_spy.rb

@@ -1,11 +1,13 @@
 class DatWorkerPool
 
   class WorkerPoolSpy
+
     attr_reader :min_workers, :max_workers, :debug
     attr_reader :work_proc, :work_items
     attr_reader :start_called
     attr_reader :shutdown_called, :shutdown_timeout
     attr_reader :on_queue_pop_callbacks, :on_queue_push_callbacks
+    attr_reader :on_worker_error_callbacks
     attr_reader :on_worker_start_callbacks, :on_worker_shutdown_callbacks
     attr_reader :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
     attr_reader :before_work_callbacks, :after_work_callbacks
@@ -25,6 +27,7 @@ class DatWorkerPool
 
       @on_queue_pop_callbacks       = []
       @on_queue_push_callbacks      = []
+      @on_worker_error_callbacks    = []
       @on_worker_start_callbacks    = []
       @on_worker_shutdown_callbacks = []
       @on_worker_sleep_callbacks    = []
@@ -33,12 +36,13 @@ class DatWorkerPool
       @after_work_callbacks         = []
     end
 
-    def worker_available?
-      @worker_available
+    def start
+      @start_called = true
     end
 
-    def queue_empty?
-      @work_items.empty?
+    def shutdown(timeout = nil)
+      @shutdown_called = true
+      @shutdown_timeout = timeout
     end
 
     def add_work(work)
@@ -53,23 +57,23 @@ class DatWorkerPool
       work
     end
 
-    def start
-      @start_called = true
+    def queue_empty?
+      @work_items.empty?
     end
 
-    def shutdown(timeout = nil)
-      @shutdown_called = true
-      @shutdown_timeout = timeout
+    def worker_available?
+      @worker_available
     end
 
-    def on_queue_pop(&block);       @on_queue_pop_callbacks << block;       end
-    def on_queue_push(&block);      @on_queue_push_callbacks << block;      end
-    def on_worker_start(&block);    @on_worker_start_callbacks << block;    end
+    def on_queue_pop(&block);       @on_queue_pop_callbacks       << block; end
+    def on_queue_push(&block);      @on_queue_push_callbacks      << block; end
+    def on_worker_error(&block);    @on_worker_error_callbacks    << block; end
+    def on_worker_start(&block);    @on_worker_start_callbacks    << block; end
     def on_worker_shutdown(&block); @on_worker_shutdown_callbacks << block; end
-    def on_worker_sleep(&block);    @on_worker_sleep_callbacks << block;    end
-    def on_worker_wakeup(&block);   @on_worker_wakeup_callbacks << block;   end
-    def before_work(&block);        @before_work_callbacks << block;        end
-    def after_work(&block);         @after_work_callbacks << block;         end
+    def on_worker_sleep(&block);    @on_worker_sleep_callbacks    << block; end
+    def on_worker_wakeup(&block);   @on_worker_wakeup_callbacks   << block; end
+    def before_work(&block);        @before_work_callbacks        << block; end
+    def after_work(&block);         @after_work_callbacks         << block; end
 
   end
 

data/test/unit/dat-worker-pool_tests.rb

@@ -11,16 +11,18 @@ class DatWorkerPool
     subject{ @work_pool }
 
     should have_readers :logger, :spawned, :queue
+    should have_readers :on_worker_error_callbacks
     should have_readers :on_worker_start_callbacks, :on_worker_shutdown_callbacks
     should have_readers :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
     should have_readers :before_work_callbacks, :after_work_callbacks
-    should have_imeths :add_work, :start, :shutdown, :despawn_worker
+    should have_imeths :add_work, :start, :shutdown
     should have_imeths :work_items, :waiting
     should have_imeths :worker_available?, :all_spawned_workers_are_busy?
     should have_imeths :reached_max_workers?
     should have_imeths :queue_empty?
     should have_imeths :on_queue_pop_callbacks, :on_queue_push_callbacks
     should have_imeths :on_queue_pop, :on_queue_push
+    should have_imeths :on_worker_error
     should have_imeths :on_worker_start, :on_worker_shutdown
     should have_imeths :on_worker_sleep, :on_worker_wakeup
     should have_imeths :before_work, :after_work
@@ -32,6 +34,7 @@ class DatWorkerPool
     end
 
     should "default its worker callbacks" do
+      assert_equal [], subject.on_worker_error_callbacks
       assert_equal [], subject.on_worker_start_callbacks
       assert_equal 1, subject.on_worker_shutdown_callbacks.size
       assert_instance_of Proc, subject.on_worker_shutdown_callbacks.first
@@ -113,21 +116,24 @@ class DatWorkerPool
   class WorkerCallbackTests < UnitTests
     desc "worker callbacks"
     setup do
-      @start_called = false
-      @shutdown_called = false
-      @sleep_called = false
-      @wakeup_called = false
+      @error_called       = false
+      @start_called       = false
+      @shutdown_called    = false
+      @sleep_called       = false
+      @wakeup_called      = false
       @before_work_called = false
-      @after_work_called = false
-
-      @work_pool = DatWorkerPool.new(1){ |work| }.tap do |p|
-        p.on_worker_start{ @start_called = true }
-        p.on_worker_shutdown{ @shutdown_called = true }
-        p.on_worker_sleep{ @sleep_called = true }
-        p.on_worker_wakeup{ @wakeup_called = true }
-        p.before_work{ @before_work_called = true }
-        p.after_work{ @after_work_called = true }
+      @after_work_called  = false
+
+      @work_pool = DatWorkerPool.new(1) do |work|
+        raise work if work == 'error'
       end
+      @work_pool.on_worker_error{ @error_called = true }
+      @work_pool.on_worker_start{ @start_called = true }
+      @work_pool.on_worker_shutdown{ @shutdown_called = true }
+      @work_pool.on_worker_sleep{ @sleep_called = true }
+      @work_pool.on_worker_wakeup{ @wakeup_called = true }
+      @work_pool.before_work{ @before_work_called = true }
+      @work_pool.after_work{ @after_work_called = true }
     end
     subject{ @work_pool }
 
@@ -137,15 +143,27 @@ class DatWorkerPool
       subject.start
       assert_true @start_called
       assert_true @sleep_called
+
+      @sleep_called = false
       assert_false @wakeup_called
       assert_false @before_work_called
       assert_false @after_work_called
-      @sleep_called = false
       subject.add_work 'work'
       assert_true @wakeup_called
       assert_true @before_work_called
       assert_true @after_work_called
       assert_true @sleep_called
+
+      @before_work_called = false
+      @after_work_called  = false
+      assert_false @before_work_called
+      assert_false @error_called
+      assert_false @after_work_called
+      subject.add_work 'error'
+      assert_true @before_work_called
+      assert_true @error_called
+      assert_false @after_work_called
+
       @wakeup_called = false
       assert_false @shutdown_called
       subject.shutdown
@@ -252,19 +270,50 @@ class DatWorkerPool
       assert_includes 'c', subject.work_items
     end
 
-    should "timeout if the workers take to long to finish" do
-      # make sure the workers haven't processed any work
+    should "allow jobs to finish by not providing a shutdown timeout" do
       assert_equal [], @finished
-      assert_raises(DatWorkerPool::TimeoutError) do
+      subject.shutdown
+      assert_includes 'a', @finished
+      assert_includes 'b', @finished
+    end
+
+    should "reraise shutdown errors in debug mode if workers take to long to finish" do
+      assert_raises(DatWorkerPool::ShutdownError) do
         subject.shutdown(0.1)
       end
     end
 
-    should "allow jobs to finish by not providing a shutdown timeout" do
+  end
+
+  class ForcedShutdownTests < UnitTests
+    desc "forced shutdown"
+    setup do
+      @mutex = Mutex.new
+      @finished = []
+      @max_workers = 2
+      # don't put leave the worker pool in debug mode
+      @work_pool = DatWorkerPool.new(1, @max_workers, false) do |work|
+        begin
+          sleep 1
+        rescue ShutdownError => error
+          @mutex.synchronize{ @finished << error }
+          raise error # re-raise it otherwise worker won't shutdown
+        end
+      end
+      @work_pool.start
+      @work_pool.add_work 'a'
+      @work_pool.add_work 'b'
+      @work_pool.add_work 'c'
+    end
+
+    should "force workers to shutdown if they take to long to finish" do
+      # make sure the workers haven't processed any work
       assert_equal [], @finished
-      subject.shutdown
-      assert_includes 'a', @finished
-      assert_includes 'b', @finished
+      subject.shutdown(0.1)
+      assert_equal @max_workers, @finished.size
+      @finished.each do |error|
+        assert_instance_of DatWorkerPool::ShutdownError, error
+      end
     end
 
   end

data/test/unit/worker_pool_spy_tests.rb

@@ -14,6 +14,7 @@ class DatWorkerPool::WorkerPoolSpy
     should have_readers :work_proc, :work_items
     should have_readers :start_called, :shutdown_called, :shutdown_timeout
     should have_readers :on_queue_pop_callbacks, :on_queue_push_callbacks
+    should have_readers :on_worker_error_callbacks
     should have_readers :on_worker_start_callbacks, :on_worker_shutdown_callbacks
     should have_readers :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
     should have_readers :before_work_callbacks, :after_work_callbacks
@@ -21,6 +22,7 @@ class DatWorkerPool::WorkerPoolSpy
     should have_imeths :worker_available?, :queue_empty?
     should have_imeths :add_work, :start, :shutdown
     should have_imeths :on_queue_pop, :on_queue_push
+    should have_imeths :on_worker_error
     should have_imeths :on_worker_start, :on_worker_shutdown
     should have_imeths :on_worker_sleep, :on_worker_wakeup
     should have_imeths :before_work, :after_work
@@ -101,6 +103,11 @@ class DatWorkerPool::WorkerPoolSpy
       subject.on_queue_push(&callback)
       assert_equal [callback], subject.on_queue_push_callbacks
 
+      assert_equal [], subject.on_worker_error_callbacks
+      callback = proc{ }
+      subject.on_worker_error(&callback)
+      assert_equal [callback], subject.on_worker_error_callbacks
+
       assert_equal [], subject.on_worker_start_callbacks
       callback = proc{ }
       subject.on_worker_start(&callback)

data/test/unit/worker_tests.rb

@@ -22,14 +22,15 @@ class DatWorkerPool::Worker
     end
     subject{ @worker }
 
-    should have_accessors :on_work
+    should have_accessors :on_work, :on_error_callbacks
     should have_accessors :on_start_callbacks, :on_shutdown_callbacks
     should have_accessors :on_sleep_callbacks, :on_wakeup_callbacks
     should have_accessors :before_work_callbacks, :after_work_callbacks
-    should have_imeths :start, :shutdown, :join, :running?
+    should have_imeths :start, :shutdown, :join, :raise, :running?
 
     should "default its callbacks" do
       worker = DatWorkerPool::Worker.new(@queue)
+      assert_equal [], worker.on_error_callbacks
       assert_equal [], worker.on_start_callbacks
       assert_equal [], worker.on_shutdown_callbacks
       assert_equal [], worker.on_sleep_callbacks
@@ -38,7 +39,7 @@ class DatWorkerPool::Worker
       assert_equal [], worker.after_work_callbacks
     end
 
-    should "start a thread with it's work loop with #start" do
+    should "start a thread with it's work loop using `start`" do
       thread = nil
       assert_nothing_raised{ thread = subject.start }
       assert_instance_of Thread, thread
@@ -55,7 +56,7 @@ class DatWorkerPool::Worker
       assert_equal [ 'one', 'two' ], @work_done
     end
 
-    should "flag itself for exiting it's work loop with #shutdown and " \
+    should "flag itself for exiting it's work loop using `shutdown` and " \
            "end it's thread once it's queue is shutdown" do
       thread = subject.start
       subject.join 0.1 # trigger the worker's thread to run, allow it to get into it's
@@ -68,53 +69,75 @@ class DatWorkerPool::Worker
       assert_not subject.running?
     end
 
+    should "raise an error on the thread using `raise`" do
+      subject.on_work = proc do |worker, work|
+        begin
+          sleep 1
+        rescue RuntimeError => error
+          @work_done << error
+          raise error
+        end
+      end
+      subject.start
+      @queue.push 'a'
+      subject.join 0.1 # trigger the worker's thread to run
+
+      exception = RuntimeError.new
+      subject.raise exception
+      assert_equal [exception], @work_done
+    end
+
   end
 
   class CallbacksTests < UnitTests
     desc "callbacks"
     setup do
       @call_counter = 0
-      @on_start_called_with = nil
-      @on_start_call_count = nil
+      @on_error_called_with    = nil
+      @on_start_called_with    = nil
+      @on_start_called_at      = nil
       @on_shutdown_called_with = nil
-      @on_shutdown_call_count = nil
-      @on_sleep_called_with = nil
-      @on_sleep_call_count = nil
-      @on_wakeup_called_with = nil
-      @on_wakeup_call_count = nil
+      @on_shutdown_called_at   = nil
+      @on_sleep_called_with    = nil
+      @on_sleep_called_at      = nil
+      @on_wakeup_called_with   = nil
+      @on_wakeup_called_at     = nil
       @before_work_called_with = nil
-      @before_work_called_at = nil
-      @after_work_called_with = nil
-      @after_work_called_at = nil
+      @before_work_called_at   = nil
+      @after_work_called_with  = nil
+      @after_work_called_at    = nil
       @worker = DatWorkerPool::Worker.new(@queue).tap do |w|
-        w.on_start_callbacks    << proc do |*args|
+        w.on_error_callbacks << proc do |*args|
+          @on_error_called_with = args
+        end
+        w.on_start_callbacks << proc do |*args|
           @on_start_called_with = args
-          @on_start_called_at = (@call_counter += 1)
+          @on_start_called_at   = (@call_counter += 1)
         end
         w.on_shutdown_callbacks << proc do |*args|
           @on_shutdown_called_with = args
-          @on_shutdown_called_at = (@call_counter += 1)
+          @on_shutdown_called_at   = (@call_counter += 1)
         end
-        w.on_sleep_callbacks    << proc do |*args|
+        w.on_sleep_callbacks << proc do |*args|
           @on_sleep_called_with = args
-          @on_sleep_called_at = (@call_counter += 1)
+          @on_sleep_called_at   = (@call_counter += 1)
         end
-        w.on_wakeup_callbacks   << proc do |*args|
+        w.on_wakeup_callbacks << proc do |*args|
           @on_wakeup_called_with = args
-          @on_wakeup_called_at = (@call_counter += 1)
+          @on_wakeup_called_at   = (@call_counter += 1)
         end
         w.before_work_callbacks << proc do |*args|
           @before_work_called_with = args
-          @before_work_called_at = (@call_counter += 1)
+          @before_work_called_at   = (@call_counter += 1)
         end
         w.after_work_callbacks << proc do |*args|
           @after_work_called_with = args
-          @after_work_called_at = (@call_counter += 1)
+          @after_work_called_at   = (@call_counter += 1)
         end
       end
     end
 
-    should "pass its self to its start, shutdown, sleep and wakupe callbacks" do
+    should "pass its self to its start, shutdown, sleep and wakeup callbacks" do
       subject.start
       @queue.push('work')
       subject.shutdown
@@ -126,7 +149,7 @@ class DatWorkerPool::Worker
       assert_equal [subject], @on_wakeup_called_with
     end
 
-    should "pass its self and work to its beofre and after work callbacks" do
+    should "pass its self and work to its before and after work callbacks" do
       subject.start
       @queue.push('work')
       subject.shutdown
@@ -151,6 +174,26 @@ class DatWorkerPool::Worker
       assert_equal 8, @on_shutdown_called_at
     end
 
+    should "call its error callbacks when an exception occurs" do
+      exception = RuntimeError.new
+      subject.on_work = proc{ raise exception }
+      thread = subject.start
+      @queue.push('work')
+      assert_equal [subject, exception, 'work'], @on_error_called_with
+      assert_true thread.alive?
+    end
+
+    should "call its error callbacks when an shutdown error occurs and reraise" do
+      exception = DatWorkerPool::ShutdownError.new
+      subject.on_work = proc{ raise exception }
+      thread = subject.start
+      @queue.push('work')
+      assert_equal [subject, exception, 'work'], @on_error_called_with
+      assert_false thread.alive?
+      # ensure the shutdown error is handled and isn't thrown when we join
+      assert_nothing_raised{ thread.join }
+    end
+
   end
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dat-worker-pool
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Collin Redding
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2015-01-02 00:00:00 Z
+date: 2015-04-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
@@ -39,11 +39,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 31
         segments: 
         - 2
-        - 12
-        version: "2.12"
+        - 14
+        version: "2.14"
   version_requirements: *id002
   type: :development
   name: assert