dat-worker-pool 0.6.0 → 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA512: 
+  data.tar.gz: c0a52d7d5c533e9d71623775137e77131dcc4eceae71e4e55085f0248983442518e394fbca0aeaf02be04f6bf7484dd139ad84f4c7e165c194057d662d7689f2
+  metadata.gz: 8c5250174e3f695c2ad78ab377e1c29973dd37295abe65cd94d3b2a266ce0171087ab1262316004591882de23497a0e24649f8c1b0c7d75ec3cbb31fdc589938
+SHA1: 
+  data.tar.gz: 4a978550e606a83e897ca45f4eaf5e8c8e945622
+  metadata.gz: 43f2dd81c2d52ce758a87a403381536899526c58

data/Gemfile

@@ -2,5 +2,4 @@ source "https://rubygems.org"
 
 gemspec
 
-gem 'rake'
 gem 'pry', "~> 0.9.0"

data/dat-worker-pool.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |gem|
   gem.version     = DatWorkerPool::VERSION
   gem.authors     = ["Collin Redding", "Kelly Redding"]
   gem.email       = ["collin.redding@me.com", "kelly@kellyredding.com"]
-  gem.description = "A simple thread pool for processing generic 'work'"
   gem.summary     = "A simple thread pool for processing generic 'work'"
+  gem.description = "A simple thread pool for processing generic 'work'"
   gem.homepage    = "http://github.com/redding/dat-worker-pool"
   gem.license     = 'MIT'
 
@@ -18,8 +18,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency("SystemTimer", ["~> 1.2"])
+  gem.add_development_dependency("assert", ["~> 2.16.1"])
 
-  gem.add_development_dependency("assert", ["~> 2.15"])
+  gem.add_dependency("much-timeout", ["~> 0.1.0"])
 
 end

data/lib/dat-worker-pool.rb

@@ -42,7 +42,7 @@ class DatWorkerPool
   end
 
   def shutdown(timeout = nil)
-    @runner.shutdown(timeout, caller)
+    @runner.shutdown(timeout)
   end
 
   def add_work(work_item)

data/lib/dat-worker-pool/runner.rb

@@ -1,5 +1,5 @@
+require 'much-timeout'
 require 'set'
-require 'system_timer'
 require 'dat-worker-pool'
 require 'dat-worker-pool/locked_object'
 
@@ -46,7 +46,7 @@ class DatWorkerPool
     # instead of letting ruby kill the threads when the process exits;
     # non-timeout errors will be re-raised so they can be caught and handled (or
     # shown when the process exits)
-    def shutdown(timeout = nil, backtrace = nil)
+    def shutdown(timeout = nil)
       log do
         timeout_message = timeout ? "#{timeout} second(s)" : "none"
         "Shutting down worker pool (timeout: #{timeout_message})"
@@ -54,15 +54,21 @@ class DatWorkerPool
       begin
         @workers.with_lock{ |m, ws| ws.each(&:dwp_signal_shutdown) }
         @queue.dwp_signal_shutdown
-        OptionalTimeout.new(timeout) do
-          @queue.dwp_shutdown
-          wait_for_workers_to_shutdown
+        MuchTimeout.just_optional_timeout(timeout, {
+          :do => proc{
+            @queue.dwp_shutdown
+            wait_for_workers_to_shutdown
+          },
+          :on_timeout => proc{
+            e = ShutdownError.new("Timed out shutting down (#{timeout} seconds).")
+            force_workers_to_shutdown(e, timeout)
+          }
+        }) do
         end
-      rescue StandardError => exception
-        force_workers_to_shutdown(exception, timeout, backtrace)
-        raise exception
-      rescue TimeoutInterruptError => exception
-        force_workers_to_shutdown(exception, timeout, backtrace)
+      rescue StandardError => err
+        e = ShutdownError.new("Errored while shutting down: #{err.inspect}")
+        force_workers_to_shutdown(e, timeout)
+        raise err
       end
       log{ "Finished shutting down" }
     end
@@ -116,16 +122,15 @@ class DatWorkerPool
       end
     end
 
-    # use an until loop instead of each to join all the workers, while we are
+    # use a while loop instead of each to join all the workers, while we are
     # joining a worker a different worker can shutdown and remove itself from
     # the `@workers` array; rescue when joining the workers, ruby will raise any
     # exceptions that aren't handled by a thread when its joined, this ensures
     # if the hard shutdown is raised and not rescued (for example, in the
     # workers ensure), then it won't cause the forced shutdown to end
     # prematurely
-    def force_workers_to_shutdown(orig_exception, timeout, backtrace)
+    def force_workers_to_shutdown(error, timeout)
       log{ "Forcing #{@workers.size} workers to shutdown" }
-      error = build_forced_shutdown_error(orig_exception, timeout, backtrace)
       while !(worker = @workers.first).nil?
         worker.dwp_raise(error)
         begin
@@ -149,34 +154,6 @@ class DatWorkerPool
       @workers.delete(worker)
     end
 
-    def build_forced_shutdown_error(orig_exception, timeout, backtrace)
-      if orig_exception.kind_of?(TimeoutInterruptError)
-        ShutdownError.new("Timed out shutting down (#{timeout} seconds).").tap do |e|
-          e.set_backtrace(backtrace) if backtrace
-        end
-      else
-        ShutdownError.new("Errored while shutting down: #{orig_exception.inspect}").tap do |e|
-          e.set_backtrace(orig_exception.backtrace)
-        end
-      end
-    end
-
-    # this needs to be an `Interrupt` to be sure we don't accidentally catch it
-    # when rescueing exceptions; in the shutdown methods we rescue any errors
-    # from `worker.join`, this will also rescue the timeout error if its a
-    # standard error and will keep it from doing a forced shutdown
-    TimeoutInterruptError = Class.new(Interrupt)
-
-    module OptionalTimeout
-      def self.new(seconds, &block)
-        if seconds
-          SystemTimer.timeout(seconds, TimeoutInterruptError, &block)
-        else
-          block.call
-        end
-      end
-    end
-
     class LoggerProxy < Struct.new(:logger)
       def runner_log(&message_block)
         self.logger.debug("[DWP] #{message_block.call}")

data/lib/dat-worker-pool/version.rb

@@ -1,3 +1,3 @@
 class DatWorkerPool
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

data/test/helper.rb

@@ -20,4 +20,11 @@ JOIN_SECONDS = 0.001
 
 require 'test/support/factory'
 
-# TODO: put test helpers here...
+# 1.8.7 backfills
+
+# Array#sample
+if !(a = Array.new).respond_to?(:sample) && a.respond_to?(:choice)
+  class Array
+    alias_method :sample, :choice
+  end
+end

data/test/support/signal_test_worker.rb

@@ -0,0 +1,71 @@
+require 'much-timeout'
+require 'dat-worker-pool/worker'
+
+module SignalTestWorker
+  def self.included(klass)
+    klass.class_eval{ include DatWorkerPool::Worker }
+  end
+
+  private
+
+  def signal_test_suite_thread
+    params[:signal_worker_mutex].synchronize do
+      params[:signal_worker_cond_var].signal
+    end
+  end
+
+  module TestHelpers
+
+    def self.included(klass)
+      klass.class_eval do
+        setup do
+          # at least 2 workers, up to 4
+          @num_workers = Factory.integer(3) + 1
+
+          @signal_worker_mutex    = Mutex.new
+          @signal_worker_cond_var = ConditionVariable.new
+
+          @worker_params = {
+            :signal_worker_mutex    => @signal_worker_mutex,
+            :signal_worker_cond_var => @signal_worker_cond_var
+          }
+        end
+      end
+    end
+
+    private
+
+    # this could loop forever so ensure it doesn't by using a timeout
+    def wait_for_workers(&block)
+      MuchTimeout.just_timeout(1, {
+        :do => proc{
+          while !block.call do
+            @signal_worker_mutex.synchronize do
+              @signal_worker_cond_var.wait(@signal_worker_mutex)
+            end
+          end
+        },
+        :on_timeout => proc{
+          raise "timed out waiting for workers" unless block.call
+        }
+      })
+    end
+
+    def wait_for_workers_to_become_available
+      wait_for_workers{ subject.available_worker_count == @num_workers }
+    end
+
+    def wait_for_workers_to_become_unavailable
+      wait_for_workers{ subject.available_worker_count == 0 }
+    end
+
+    def wait_for_a_worker_to_become_available
+      wait_for_workers{ subject.available_worker_count != 0 }
+    end
+
+    def wait_for_a_worker_to_become_unavailable
+      wait_for_workers{ subject.available_worker_count != @num_workers }
+    end
+
+  end
+end

data/test/system/dat-worker-pool_tests.rb

@@ -1,58 +1,24 @@
 require 'assert'
 require 'dat-worker-pool'
 
-require 'timeout'
 require 'dat-worker-pool/locked_object'
 require 'dat-worker-pool/worker'
+require 'test/support/signal_test_worker'
 
 class DatWorkerPool
 
   class SystemTests < Assert::Context
+    include SignalTestWorker::TestHelpers
+
     desc "DatWorkerPool"
-    setup do
-      # at least 2 workers, up to 4
-      @num_workers   = Factory.integer(3) + 1
-      @mutex         = Mutex.new
-      @cond_var      = ConditionVariable.new
-      @worker_params = {
-        :mutex    => @mutex,
-        :cond_var => @cond_var
-      }
-    end
     subject{ @worker_pool }
 
-    # this could loop forever so ensure it doesn't by using a timeout; use
-    # timeout instead of system timer because system timer is paranoid about a
-    # deadlock even though its intended to prevent the deadlock because it times
-    # out the block
-    def wait_for_workers(&block)
-      Timeout.timeout(1) do
-        @mutex.synchronize{ @cond_var.wait(@mutex) } while !block.call
-      end
-    end
-
-    def wait_for_workers_to_become_available
-      wait_for_workers{ @worker_pool.available_worker_count == @num_workers }
-    end
-
-    def wait_for_workers_to_become_unavailable
-      wait_for_workers{ @worker_pool.available_worker_count == 0 }
-    end
-
-    def wait_for_a_worker_to_become_available
-      wait_for_workers{ @worker_pool.available_worker_count != 0 }
-    end
-
-    def wait_for_a_worker_to_become_unavailable
-      wait_for_workers{ @worker_pool.available_worker_count != @num_workers }
-    end
-
   end
 
   class StartAddProcessAndShutdownTests < SystemTests
     setup do
       @worker_class = Class.new do
-        include SystemTestWorker
+        include SignalTestWorker
         def work!(number)
           params[:results].push(number * 100)
           signal_test_suite_thread
@@ -88,17 +54,20 @@ class DatWorkerPool
   class WorkerAvailabilityTests < SystemTests
     setup do
       @worker_class = Class.new do
-        include SystemTestWorker
+        include SignalTestWorker
         on_available{   signal_test_suite_thread }
         on_unavailable{ signal_test_suite_thread }
 
         # this allows controlling how many workers are available and unavailable
         # the worker will be unavailable until we signal it
         def work!(work_item)
+          params[:working].add(self)
           mutex, cond_var = work_item
           mutex.synchronize{ cond_var.wait(mutex) }
+          params[:working].remove(self)
         end
       end
+      @working       = LockedSet.new
       @work_mutex    = Mutex.new
       @work_cond_var = ConditionVariable.new
       @work_item     = [@work_mutex, @work_cond_var]
@@ -106,7 +75,7 @@ class DatWorkerPool
       @worker_pool = DatWorkerPool.new(@worker_class, {
         :num_workers   => @num_workers,
         :logger        => TEST_LOGGER,
-        :worker_params => @worker_params
+        :worker_params => @worker_params.merge(:working => @working)
       })
       @worker_pool.start
     end
@@ -131,7 +100,7 @@ class DatWorkerPool
       # make the rest of the workers unavailable
       (@num_workers - 1).times{ subject.add_work(@work_item) }
 
-      wait_for_workers_to_become_unavailable
+      wait_for_workers{ @working.size == @num_workers }
       assert_equal 0, subject.available_worker_count
       assert_false subject.worker_available?
 
@@ -155,7 +124,7 @@ class DatWorkerPool
   class WorkerCallbackTests < SystemTests
     setup do
       @worker_class = Class.new do
-        include SystemTestWorker
+        include SignalTestWorker
 
         on_start{    params[:callbacks_called][:on_start]    = true }
         on_shutdown{ params[:callbacks_called][:on_shutdown] = true }
@@ -269,7 +238,7 @@ class DatWorkerPool
   class ShutdownSystemTests < SystemTests
     setup do
       @worker_class = Class.new do
-        include SystemTestWorker
+        include SignalTestWorker
         on_available{   signal_test_suite_thread }
         on_unavailable{ signal_test_suite_thread }
 
@@ -280,12 +249,15 @@ class DatWorkerPool
         # this allows controlling how long a worker takes to finish processing
         # the work item
         def work!(work_item)
+          params[:working].add(self)
           params[:work_mutex].synchronize do
             params[:work_cond_var].wait(params[:work_mutex])
           end
           params[:finished].push(work_item)
+          params[:working].remove(self)
         end
       end
+      @working       = LockedSet.new
       @work_mutex    = Mutex.new
       @work_cond_var = ConditionVariable.new
       @finished      = LockedArray.new
@@ -295,6 +267,7 @@ class DatWorkerPool
         :num_workers   => @num_workers,
         :logger        => TEST_LOGGER,
         :worker_params => @worker_params.merge({
+          :working       => @working,
           :work_mutex    => @work_mutex,
           :work_cond_var => @work_cond_var,
           :finished      => @finished,
@@ -308,7 +281,7 @@ class DatWorkerPool
       # add 1 more work item than we have workers to handle it
       @work_items = (@num_workers + 1).times.map{ Factory.string }
       @work_items.each{ |wi| @worker_pool.add_work(wi) }
-      wait_for_workers_to_become_unavailable
+      wait_for_workers{ @working.size == @num_workers }
     end
     teardown do
       # ensure we wakeup any workers still stuck in their `work!`
@@ -351,17 +324,12 @@ class DatWorkerPool
       assert_false shutdown_thread.alive?
     end
 
-    should "allow any work that has been picked up to finish processing " \
+    should "not allow any work that has been picked up to finish processing " \
            "when forced to shutdown because it timed out" do
       assert_true @finished.empty?
       assert_true @errored.empty?
 
-      # start the shutdown in a thread, this will hang until the timeout
-      # finishes; this is required otherwise system timer will think we are
-      # triggering a deadlock (it's not a deadlock because of the timeout)
-      shutdown_thread = Thread.new{ subject.shutdown(0) }
-      shutdown_thread.join(JOIN_SECONDS)
-      assert_equal 'sleep', shutdown_thread.status
+      subject.shutdown(0)
 
       # wait for the workers to get forced to exit
       wait_for_workers{ @errored.size == @num_workers }
@@ -378,22 +346,8 @@ class DatWorkerPool
         assert_instance_of ShutdownError, exception
         assert_includes work_item, @work_items[0, @num_workers]
       end
-
-      # ensure the shutdown exits
-      shutdown_thread.join
-      assert_false shutdown_thread.alive?
-    end
-
-  end
-
-  module SystemTestWorker
-    def self.included(klass)
-      klass.class_eval{ include DatWorkerPool::Worker }
     end
 
-    def signal_test_suite_thread
-      params[:mutex].synchronize{ params[:cond_var].signal }
-    end
   end
 
 end

data/test/unit/dat-worker-pool_tests.rb

@@ -1,7 +1,6 @@
 require 'assert'
 require 'dat-worker-pool'
 
-require 'system_timer'
 require 'dat-worker-pool/queue'
 require 'dat-worker-pool/runner'
 require 'dat-worker-pool/worker'
@@ -97,8 +96,6 @@ class DatWorkerPool
       subject.shutdown
       assert_true @runner_spy.shutdown_called
       assert_nil @runner_spy.shutdown_timeout
-      exp = "test/unit/dat-worker-pool_tests.rb"
-      assert_match exp, @runner_spy.shutdown_backtrace.first
 
       timeout = Factory.integer
       subject.shutdown(timeout)
@@ -118,7 +115,7 @@ class DatWorkerPool
     should "raise an argument error if given an invalid number of workers" do
       assert_raises(ArgumentError) do
         @worker_pool_class.new(@worker_class, {
-          :num_workers  => [0, (Factory.integer * -1)].choice
+          :num_workers  => [0, (Factory.integer * -1)].sample
         })
       end
     end
@@ -172,14 +169,13 @@ class DatWorkerPool
   class RunnerSpy < DatWorkerPool::Runner
     attr_accessor :args
     attr_reader :start_called, :shutdown_called
-    attr_reader :shutdown_timeout, :shutdown_backtrace
+    attr_reader :shutdown_timeout
 
     def initialize
       super({})
       @start_called       = false
       @shutdown_called    = false
       @shutdown_timeout   = nil
-      @shutdown_backtrace = nil
     end
 
     def start
@@ -187,11 +183,10 @@ class DatWorkerPool
       @start_called = true
     end
 
-    def shutdown(timeout, backtrace)
+    def shutdown(timeout)
       @args[:queue].dwp_shutdown
-      @shutdown_called    = true
-      @shutdown_timeout   = timeout
-      @shutdown_backtrace = backtrace
+      @shutdown_called  = true
+      @shutdown_timeout = timeout
     end
   end
 

data/test/unit/default_queue_tests.rb

@@ -162,6 +162,7 @@ class DatWorkerPool::DefaultQueue
 
       value = Factory.string
       @queue.dwp_push(value)
+      subject.join(JOIN_SECONDS)
 
       assert_not subject.alive?
       assert_equal value, subject['popped_value']
@@ -176,6 +177,7 @@ class DatWorkerPool::DefaultQueue
       # sleeps and grabs the lock and work item before the thread being woken
       # up
       @work_items.with_lock{ @cond_var_spy.signal }
+      subject.join(JOIN_SECONDS)
 
       assert_equal 'sleep', subject.status
       assert_equal 2, @cond_var_spy.wait_call_count
@@ -186,6 +188,7 @@ class DatWorkerPool::DefaultQueue
       assert_equal 1, @cond_var_spy.wait_call_count
 
       @queue.dwp_shutdown
+      subject.join(JOIN_SECONDS)
 
       assert_not subject.alive?
       assert_equal 1, @cond_var_spy.wait_call_count
@@ -207,6 +210,7 @@ class DatWorkerPool::DefaultQueue
       # accesses the array directly and pushes an item on it
       @work_items.push(Factory.string)
       @queue.dwp_shutdown
+      subject.join(JOIN_SECONDS)
 
       assert_not subject.alive?
       assert_nil subject['popped_value']

data/test/unit/runner_tests.rb

@@ -1,11 +1,15 @@
 require 'assert'
 require 'dat-worker-pool/runner'
 
+require 'much-timeout'
 require 'dat-worker-pool/default_queue'
+require 'test/support/signal_test_worker'
 
 class DatWorkerPool::Runner
 
   class UnitTests < Assert::Context
+    include SignalTestWorker::TestHelpers
+
     desc "DatWorkerPool::Runner"
     setup do
       @runner_class = DatWorkerPool::Runner
@@ -17,12 +21,13 @@ class DatWorkerPool::Runner
   class InitTests < UnitTests
     desc "when init"
     setup do
-      # at least 2 workers, up to 4
-      @num_workers   = Factory.integer(3) + 1
-      @logger        = TEST_LOGGER || Logger.new("/dev/null")
-      @queue         = DatWorkerPool::DefaultQueue.new
-      @worker_class  = TestWorker
-      @worker_params = { Factory.string => Factory.string }
+      @logger       = TEST_LOGGER || Logger.new("/dev/null")
+      @queue        = DatWorkerPool::DefaultQueue.new
+      @worker_class = TestWorker
+
+      @worker_params.merge!({
+        Factory.string => Factory.string
+      })
 
       @workers = DatWorkerPool::LockedArray.new
       Assert.stub(DatWorkerPool::LockedArray, :new){ @workers }
@@ -73,24 +78,6 @@ class DatWorkerPool::Runner
       assert_equal @workers.values, subject.workers
     end
 
-    should "start its queue when its started" do
-      assert_false @queue.running?
-      subject.start
-      assert_true @queue.running?
-    end
-
-    should "build and add workers when its started" do
-      subject.start
-
-      assert_equal @num_workers, subject.workers.size
-      subject.workers.each_with_index do |worker, n|
-        assert_equal subject, worker.dwp_runner
-        assert_equal @queue,  worker.dwp_queue
-        assert_equal n + 1,   worker.dwp_number
-        assert_true worker.dwp_running?
-      end
-    end
-
     should "allow making workers available/unavailable" do
       worker = @worker_class.new(@runner, @queue, Factory.integer(10))
 
@@ -142,25 +129,45 @@ class DatWorkerPool::Runner
 
   end
 
-  class ShutdownSetupTests < InitTests
-    desc "and started and shutdown"
+  class StartTests < InitTests
+    desc "and started"
+
+    should "start its queue" do
+      assert_false @queue.running?
+      subject.start
+      assert_true @queue.running?
+    end
+
+    should "build and add workers" do
+      subject.start
+      wait_for_workers_to_become_available
+
+      assert_equal @num_workers, subject.workers.size
+      subject.workers.each_with_index do |worker, n|
+        assert_equal subject, worker.dwp_runner
+        assert_equal @queue,  worker.dwp_queue
+        assert_equal n + 1,   worker.dwp_number
+        assert_true worker.dwp_running?
+      end
+    end
 
   end
 
-  class ShutdownTests < ShutdownSetupTests
+  class ShutdownTests < StartTests
+    desc "and shutdown"
     setup do
       @timeout_seconds         = nil
       @optional_timeout_called = false
       # this acts as a spy but also keeps the shutdown from ever timing out
-      Assert.stub(OptionalTimeout, :new) do |secs, &block|
+      Assert.stub(MuchTimeout, :just_optional_timeout) do |secs, args|
         @timeout_seconds         = secs
         @optional_timeout_called = true
-        block.call
+        args[:do].call
       end
 
-      @options[:worker_class] = ShutdownSpyWorker
-      @runner = @runner_class.new(@options)
       @runner.start
+      wait_for_workers_to_become_available
+
       # we need a reference to the workers, the runners workers will get removed
       # as they shutdown
       @running_workers = @runner.workers.dup
@@ -182,7 +189,10 @@ class DatWorkerPool::Runner
       @running_workers.each do |worker|
         assert_false worker.dwp_shutdown?
       end
+
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
+      wait_for_workers_to_become_unavailable
+
       @running_workers.each do |worker|
         assert_true worker.dwp_shutdown?
       end
@@ -198,15 +208,21 @@ class DatWorkerPool::Runner
       @running_workers.each do |worker|
         assert_false worker.join_called
       end
+
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
+      wait_for_workers_to_become_unavailable
+
       @running_workers.each do |worker|
         assert_true worker.join_called
       end
     end
 
     should "join all workers even if one raises an error when joined" do
-      @running_workers.choice.join_error = Factory.exception
+      @running_workers.sample.join_error = Factory.exception
+
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
+      wait_for_workers_to_become_unavailable
+
       @running_workers.each do |worker|
         assert_true worker.join_called
       end
@@ -214,7 +230,10 @@ class DatWorkerPool::Runner
 
     should "remove workers as they finish" do
       assert_false subject.workers.empty?
+
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
+      wait_for_workers_to_become_unavailable
+
       assert_true subject.workers.empty?
     end
 
@@ -223,13 +242,21 @@ class DatWorkerPool::Runner
 
       assert_false subject.workers.empty?
       assert_false @available_workers_spy.empty?
+
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
+      wait_for_workers_to_become_unavailable
+
       assert_true subject.workers.empty?
       assert_true @available_workers_spy.empty?
     end
 
     should "force its workers to shutdown if a timeout error occurs" do
-      Assert.stub(OptionalTimeout, :new){ raise TimeoutInterruptError }
+      Assert.stub(MuchTimeout, :just_optional_timeout) do |secs, args|
+        args[:on_timeout].call # force an immediate timeout
+      end
+
+      @num_workers.times{ @queue.dwp_push(:hang) }
+      wait_for_workers_to_become_unavailable
       subject.shutdown(Factory.integer)
 
       @running_workers.each do |worker|
@@ -244,6 +271,9 @@ class DatWorkerPool::Runner
       queue_exception = Factory.exception
       Assert.stub(@queue, :dwp_shutdown){ raise queue_exception }
 
+      @num_workers.times{ @queue.dwp_push(:hang) }
+      wait_for_workers_to_become_unavailable
+
       caught_exception = nil
       begin
         subject.shutdown(Factory.integer)
@@ -260,15 +290,24 @@ class DatWorkerPool::Runner
     end
 
     should "force shutdown all of its workers even if one raises an error when joining" do
-      Assert.stub(OptionalTimeout, :new){ raise TimeoutInterruptError }
-      error_class = Factory.boolean ? DatWorkerPool::ShutdownError : RuntimeError
-      @running_workers.choice.join_error = Factory.exception(error_class)
+      Assert.stub(MuchTimeout, :just_optional_timeout) do |secs, args|
+        args[:on_timeout].call # force an immediate timeout
+      end
+      error_class  = Factory.boolean ? DatWorkerPool::ShutdownError : RuntimeError
+      error_worker = @running_workers.sample
+      error_worker.join_error = Factory.exception(error_class)
+
+      @num_workers.times{ @queue.dwp_push(:hang) }
+      wait_for_workers_to_become_unavailable
       subject.shutdown(Factory.boolean ? Factory.integer : nil)
 
-      @running_workers.each do |worker|
+      (@running_workers - [error_worker]).each do |worker|
         assert_instance_of DatWorkerPool::ShutdownError, worker.raised_error
         assert_true worker.join_called
       end
+      assert_instance_of error_class, error_worker.raised_error
+      assert_true error_worker.join_called
+
       assert_true subject.workers.empty?
       assert_true @available_workers_spy.empty?
     end
@@ -318,48 +357,46 @@ class DatWorkerPool::Runner
   end
 
   class TestWorker
-    include DatWorkerPool::Worker
+    include SignalTestWorker
+
+    attr_reader :join_called
+    attr_accessor :join_error, :raised_error
 
     # for testing what is passed to the worker
     attr_reader :dwp_runner, :dwp_queue
-  end
 
-  FakeWorker = Struct.new(:dwp_number)
+    on_available{   signal_test_suite_thread }
+    on_unavailable{ signal_test_suite_thread }
 
-  class ShutdownSpyWorker < TestWorker
-    attr_reader :join_called
-    attr_accessor :join_error, :raised_error
+    on_shutdown{ raise self.join_error if self.join_error }
 
-    # this pauses the shutdown so we can test that join or raise are called
-    # depending if we are doing a standard or forced shutdown; otherwise the
-    # worker threads can exit before join or raise ever gets called on them and
-    # then there is nothing to test
-    on_shutdown{ wait_for_join_or_raise }
+    on_error{ |e, wi| self.raised_error = e }
 
     def initialize(*args)
       super
-      @mutex        = Mutex.new
-      @cond_var     = ConditionVariable.new
-      @join_called  = false
-      @join_error   = nil
-      @raised_error = nil
-    end
+      @join_called = false
+      @join_error  = nil
 
-    def dwp_join(*args)
-      @join_called = true
-      raise @join_error if @join_error
-      @mutex.synchronize{ @cond_var.broadcast }
+      @hang_mutex    = Mutex.new
+      @hang_cond_var = ConditionVariable.new
     end
 
-    def dwp_raise(error)
-      @raised_error = error
+    def work!(work_item)
+      case(work_item)
+      when :hang
+        MuchTimeout.timeout(1) do
+          @hang_mutex.synchronize{ @hang_cond_var.wait(@hang_mutex) }
+        end
+      end
     end
 
-    private
-
-    def wait_for_join_or_raise
-      @mutex.synchronize{ @cond_var.wait(@mutex) }
+    def dwp_join(*args)
+      @join_called = true
+      super
     end
+
   end
 
+  FakeWorker = Struct.new(:dwp_number)
+
 end

data/test/unit/worker_tests.rb

@@ -1,7 +1,7 @@
 require 'assert'
 require 'dat-worker-pool/worker'
 
-require 'system_timer'
+require 'much-timeout'
 require 'dat-worker-pool/default_queue'
 require 'dat-worker-pool/runner'
 require 'test/support/thread_spies'
@@ -125,9 +125,18 @@ module DatWorkerPool::Worker
       })
       @number = Factory.integer(10)
 
-      @thread_spy = ThreadSpy.new
+      # we only want to stub the first `Thread.new` call, not all calls;
+      # stubbing all threads can generate unexpected behavior and specifically
+      # causes `MuchTimeout` to not work correctly
+      @worker_thread_built = false
+      @thread_spy          = ThreadSpy.new
       Assert.stub(Thread, :new) do |&block|
-        @thread_spy.tap{ |s| s.block = block }
+        if !@worker_thread_built
+          @worker_thread_built = true
+          @thread_spy.tap{ |s| s.block = block }
+        else
+          Assert.stub_send(Thread, :new, &block)
+        end
       end
 
       @available_worker = nil
@@ -245,7 +254,7 @@ module DatWorkerPool::Worker
 
     should "make itself available/unavailable and run callbacks if it errors" do
       # these are the only errors that could interfere with it
-      error_method = [:on_unavailable_error, :work_error].choice
+      error_method = [:on_unavailable_error, :work_error].sample
       exception = Factory.exception
       subject.send("#{error_method}=", exception)
 
@@ -295,7 +304,7 @@ module DatWorkerPool::Worker
 
     should "run its on-error callbacks if it errors while starting" do
       exception = Factory.exception
-      error_method = [:on_start_error, :on_available_error].choice
+      error_method = [:on_start_error, :on_available_error].sample
       subject.send("#{error_method}=", exception)
 
       subject.dwp_start
@@ -307,7 +316,7 @@ module DatWorkerPool::Worker
 
     should "stop its thread if it errors while starting" do
       exception = Factory.exception
-      error_method = [:on_start_error, :on_available_error].choice
+      error_method = [:on_start_error, :on_available_error].sample
       subject.send("#{error_method}=", exception)
 
       subject.dwp_start
@@ -319,7 +328,7 @@ module DatWorkerPool::Worker
 
     should "run its on-error callbacks if it errors while shutting down" do
       exception = Factory.exception
-      error_method = [:on_shutdown_error, :on_unavailable_error].choice
+      error_method = [:on_shutdown_error, :on_unavailable_error].sample
       subject.send("#{error_method}=", exception)
 
       subject.dwp_start
@@ -401,7 +410,7 @@ module DatWorkerPool::Worker
       subject.dwp_start
       wait_for_worker_to_be_available
 
-      error_method = ERROR_METHODS.reject{ |e| e == :on_available_error }.choice
+      error_method = ERROR_METHODS.reject{ |e| e == :on_available_error }.sample
       subject.send("#{error_method}=", exception)
       work_item = Factory.string
       @queue.dwp_push(work_item)
@@ -544,17 +553,14 @@ module DatWorkerPool::Worker
       :after_work_error
     ].freeze
     def setup_work_loop_to_raise_exception(exception)
-      error_method = ERROR_METHODS.choice
+      error_method = ERROR_METHODS.sample
       @worker.send("#{error_method}=", exception)
       error_method
     end
 
-    # this could loop forever so ensure it doesn't by using a timeout; use
-    # timeout instead of system timer because system timer is paranoid about a
-    # deadlock even though its intended to prevent the deadlock because it times
-    # out the block
+    # this could loop forever so ensure it doesn't by using a timeout
     def wait_for_worker(&block)
-      Timeout.timeout(1) do
+      MuchTimeout.timeout(1) do
         @mutex.synchronize{ @cond_var.wait(@mutex) } while !block.call
       end
     end
@@ -587,7 +593,7 @@ module DatWorkerPool::Worker
 
     def wait_for_worker_thread_to_stop
       return unless @worker.dwp_thread_alive?
-      Timeout.timeout(1){ @worker.dwp_join }
+      MuchTimeout.timeout(1){ @worker.dwp_join }
     end
 
     # this is needed because errors will be re-raised when the thread is joined

metadata

@@ -1,13 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: dat-worker-pool
 version: !ruby/object:Gem::Version 
-  hash: 7
-  prerelease: 
-  segments: 
-  - 0
-  - 6
-  - 0
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors: 
 - Collin Redding
@@ -16,38 +10,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2015-11-24 00:00:00 Z
+date: 2016-06-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
+  name: assert
+  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 1
-        - 2
-        version: "1.2"
-  type: :runtime
-  name: SystemTimer
+        version: 2.16.1
+  type: :development
   version_requirements: *id001
-  prerelease: false
 - !ruby/object:Gem::Dependency 
+  name: much-timeout
+  prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 2
-        - 15
-        version: "2.15"
-  type: :development
-  name: assert
+        version: 0.1.0
+  type: :runtime
   version_requirements: *id002
-  prerelease: false
 description: A simple thread pool for processing generic 'work'
 email: 
 - collin.redding@me.com
@@ -61,9 +45,8 @@ extra_rdoc_files: []
 files: 
 - .gitignore
 - Gemfile
-- LICENSE.txt
+- LICENSE
 - README.md
-- Rakefile
 - bench/report.rb
 - bench/report.txt
 - dat-worker-pool.gemspec
@@ -78,6 +61,7 @@ files:
 - log/.gitkeep
 - test/helper.rb
 - test/support/factory.rb
+- test/support/signal_test_worker.rb
 - test/support/thread_spies.rb
 - test/system/dat-worker-pool_tests.rb
 - test/unit/dat-worker-pool_tests.rb
@@ -91,39 +75,33 @@ files:
 homepage: http://github.com/redding/dat-worker-pool
 licenses: 
 - MIT
+metadata: {}
+
 post_install_message: 
 rdoc_options: []
 
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
-  - - ">="
+  - &id003 
+    - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  - *id003
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.6.4
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A simple thread pool for processing generic 'work'
 test_files: 
 - test/helper.rb
 - test/support/factory.rb
+- test/support/signal_test_worker.rb
 - test/support/thread_spies.rb
 - test/system/dat-worker-pool_tests.rb
 - test/unit/dat-worker-pool_tests.rb

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"