dat-worker-pool 0.5.0 → 0.6.0

data/lib/dat-worker-pool/worker_pool_spy.rb

@@ -1,80 +1,66 @@
+require 'dat-worker-pool/worker'
+
 class DatWorkerPool
 
   class WorkerPoolSpy
 
-    attr_reader :min_workers, :max_workers, :debug
-    attr_reader :work_proc, :work_items
-    attr_reader :start_called
-    attr_reader :shutdown_called, :shutdown_timeout
-    attr_reader :on_queue_pop_callbacks, :on_queue_push_callbacks
-    attr_reader :on_worker_error_callbacks
-    attr_reader :on_worker_start_callbacks, :on_worker_shutdown_callbacks
-    attr_reader :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
-    attr_reader :before_work_callbacks, :after_work_callbacks
-    attr_accessor :worker_available
+    attr_reader :logger, :queue
+    attr_reader :options, :num_workers, :worker_class, :worker_params
+    attr_reader :start_called, :shutdown_called, :shutdown_timeout
+    attr_accessor :available_worker_count, :worker_available
+
+    def initialize(worker_class, options = nil)
+      @worker_class = worker_class
+      if !@worker_class.kind_of?(Class) || !@worker_class.include?(DatWorkerPool::Worker)
+        raise ArgumentError, "worker class must include `#{DatWorkerPool::Worker}`"
+      end
+
+      @options      = options || {}
+      @num_workers  = (@options[:num_workers] || DEFAULT_NUM_WORKERS).to_i
+      if @num_workers && @num_workers < MIN_WORKERS
+        raise ArgumentError, "number of workers must be at least #{MIN_WORKERS}"
+      end
 
-    def initialize(min = 0, max = 1, debug = false, &block)
-      @min_workers  = min
-      @max_workers  = max
-      @debug        = debug
-      @work_proc    = block
+      @queue = @options[:queue] || begin
+        require 'dat-worker-pool/default_queue'
+        DatWorkerPool::DefaultQueue.new
+      end
 
-      @worker_available = false
-      @work_items = []
-      @start_called = false
-      @shutdown_called  = false
-      @shutdown_timeout = nil
+      @logger        = @options[:logger]
+      @worker_params = @options[:worker_params]
 
-      @on_queue_pop_callbacks       = []
-      @on_queue_push_callbacks      = []
-      @on_worker_error_callbacks    = []
-      @on_worker_start_callbacks    = []
-      @on_worker_shutdown_callbacks = []
-      @on_worker_sleep_callbacks    = []
-      @on_worker_wakeup_callbacks   = []
-      @before_work_callbacks        = []
-      @after_work_callbacks         = []
+      @available_worker_count = 0
+      @worker_available       = false
+      @start_called           = false
+      @shutdown_called        = false
+      @shutdown_timeout       = nil
     end
 
     def start
       @start_called = true
+      @queue.dwp_start
     end
 
     def shutdown(timeout = nil)
-      @shutdown_called = true
+      @shutdown_called  = true
       @shutdown_timeout = timeout
+      @queue.dwp_shutdown
     end
 
-    def add_work(work)
-      return unless work
-      @work_items << work
-      @on_queue_push_callbacks.each(&:call)
+    def add_work(work_item)
+      return if work_item.nil?
+      @queue.dwp_push(work_item)
     end
+    alias :push :add_work
 
-    def pop_work
-      work = @work_items.shift
-      @on_queue_pop_callbacks.each(&:call)
-      work
-    end
-
-    def queue_empty?
-      @work_items.empty?
+    def work_items
+      @queue.work_items
     end
 
     def worker_available?
-      @worker_available
+      !!@worker_available
     end
 
-    def on_queue_pop(&block);       @on_queue_pop_callbacks       << block; end
-    def on_queue_push(&block);      @on_queue_push_callbacks      << block; end
-    def on_worker_error(&block);    @on_worker_error_callbacks    << block; end
-    def on_worker_start(&block);    @on_worker_start_callbacks    << block; end
-    def on_worker_shutdown(&block); @on_worker_shutdown_callbacks << block; end
-    def on_worker_sleep(&block);    @on_worker_sleep_callbacks    << block; end
-    def on_worker_wakeup(&block);   @on_worker_wakeup_callbacks   << block; end
-    def before_work(&block);        @before_work_callbacks        << block; end
-    def after_work(&block);         @after_work_callbacks         << block; end
-
   end
 
 end

data/test/helper.rb

@@ -7,4 +7,17 @@ $LOAD_PATH.unshift(File.expand_path("../..", __FILE__))
 # require pry for debugging (`binding.pry`)
 require 'pry'
 
+require 'pathname'
+ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
+
+require 'logger'
+TEST_LOGGER = if ENV['DEBUG']
+  # don't show datetime in the logs
+  Logger.new(ROOT_PATH.join("log/test.log")).tap{ |l| l.datetime_format = '' }
+end
+
+JOIN_SECONDS = 0.001
+
+require 'test/support/factory'
+
 # TODO: put test helpers here...

data/test/support/factory.rb

@@ -0,0 +1,15 @@
+require 'assert/factory'
+
+module Factory
+  extend Assert::Factory
+
+  def self.exception(klass = nil, message = nil)
+    klass ||= StandardError
+    message ||= Factory.text
+    exception = nil
+    begin; raise(klass, message); rescue klass => exception; end
+    exception.set_backtrace(nil) if Factory.boolean
+    exception
+  end
+
+end

data/test/support/thread_spies.rb

@@ -0,0 +1,83 @@
+require 'thread'
+
+class ThreadSpy
+  attr_reader :join_seconds, :join_called
+  attr_reader :raised_exception
+
+  def initialize(&block)
+    @block    = block
+    @mutex    = Mutex.new
+    @cond_var = ConditionVariable.new
+
+    @join_seconds     = nil
+    @join_called      = false
+    @raised_exception = nil
+
+    @thread = Thread.new do
+      @mutex.synchronize{ @cond_var.wait(@mutex) } if @block.nil?
+      @block.call
+    end
+  end
+
+  def block=(new_block)
+    @block = new_block
+    @mutex.synchronize{ @cond_var.signal }
+  end
+
+  def join(seconds = nil)
+    @join_seconds = seconds
+    @join_called  = true
+    @thread.join(seconds)
+  end
+
+  def raise(exception)
+    @raised_exception = exception
+    @thread.raise(exception)
+  end
+
+  def status; @thread.status; end
+  def alive?; @thread.alive?; end
+end
+
+class MutexSpy < Mutex
+  attr_accessor :synchronize_called
+
+  def initialize
+    @synchronize_called = false
+    super
+  end
+
+  def synchronize
+    @synchronize_called = true
+    super
+  end
+end
+
+class ConditionVariableSpy < ConditionVariable
+  attr_reader :signal_called, :broadcast_called
+  attr_reader :wait_called_on, :wait_call_count
+
+  def initialize
+    @signal_called    = false
+    @broadcast_called = false
+    @wait_called_on   = nil
+    @wait_call_count  = 0
+    super
+  end
+
+  def signal
+    @signal_called = true
+    super
+  end
+
+  def broadcast
+    @broadcast_called = true
+    super
+  end
+
+  def wait(mutex)
+    @wait_called_on  = mutex
+    @wait_call_count += 1
+    super
+  end
+end

data/test/system/dat-worker-pool_tests.rb

@@ -0,0 +1,399 @@
+require 'assert'
+require 'dat-worker-pool'
+
+require 'timeout'
+require 'dat-worker-pool/locked_object'
+require 'dat-worker-pool/worker'
+
+class DatWorkerPool
+
+  class SystemTests < Assert::Context
+    desc "DatWorkerPool"
+    setup do
+      # at least 2 workers, up to 4
+      @num_workers   = Factory.integer(3) + 1
+      @mutex         = Mutex.new
+      @cond_var      = ConditionVariable.new
+      @worker_params = {
+        :mutex    => @mutex,
+        :cond_var => @cond_var
+      }
+    end
+    subject{ @worker_pool }
+
+    # this could loop forever so ensure it doesn't by using a timeout; use
+    # timeout instead of system timer because system timer is paranoid about a
+    # deadlock even though its intended to prevent the deadlock because it times
+    # out the block
+    def wait_for_workers(&block)
+      Timeout.timeout(1) do
+        @mutex.synchronize{ @cond_var.wait(@mutex) } while !block.call
+      end
+    end
+
+    def wait_for_workers_to_become_available
+      wait_for_workers{ @worker_pool.available_worker_count == @num_workers }
+    end
+
+    def wait_for_workers_to_become_unavailable
+      wait_for_workers{ @worker_pool.available_worker_count == 0 }
+    end
+
+    def wait_for_a_worker_to_become_available
+      wait_for_workers{ @worker_pool.available_worker_count != 0 }
+    end
+
+    def wait_for_a_worker_to_become_unavailable
+      wait_for_workers{ @worker_pool.available_worker_count != @num_workers }
+    end
+
+  end
+
+  class StartAddProcessAndShutdownTests < SystemTests
+    setup do
+      @worker_class = Class.new do
+        include SystemTestWorker
+        def work!(number)
+          params[:results].push(number * 100)
+          signal_test_suite_thread
+        end
+      end
+
+      # at least 5 work items, up to 10
+      @work_items = (Factory.integer(5) + 5).times.map{ Factory.integer(10) }
+      @results    = LockedArray.new
+
+      @worker_pool = DatWorkerPool.new(@worker_class, {
+        :num_workers   => @num_workers,
+        :logger        => TEST_LOGGER,
+        :worker_params => @worker_params.merge(:results => @results)
+      })
+    end
+
+    should "be able to start, add work, process it and shutdown" do
+      subject.start
+      @work_items.each{ |work_item| subject.add_work(work_item) }
+
+      wait_for_workers{ @results.size == @work_items.size }
+      subject.shutdown(0)
+
+      assert_equal @work_items.size, @results.size
+      @work_items.each do |number|
+        assert_includes number * 100, @results.values
+      end
+    end
+
+  end
+
+  class WorkerAvailabilityTests < SystemTests
+    setup do
+      @worker_class = Class.new do
+        include SystemTestWorker
+        on_available{   signal_test_suite_thread }
+        on_unavailable{ signal_test_suite_thread }
+
+        # this allows controlling how many workers are available and unavailable
+        # the worker will be unavailable until we signal it
+        def work!(work_item)
+          mutex, cond_var = work_item
+          mutex.synchronize{ cond_var.wait(mutex) }
+        end
+      end
+      @work_mutex    = Mutex.new
+      @work_cond_var = ConditionVariable.new
+      @work_item     = [@work_mutex, @work_cond_var]
+
+      @worker_pool = DatWorkerPool.new(@worker_class, {
+        :num_workers   => @num_workers,
+        :logger        => TEST_LOGGER,
+        :worker_params => @worker_params
+      })
+      @worker_pool.start
+    end
+    teardown do
+      # ensure we wakeup any workers still stuck in their `work!`
+      @work_mutex.synchronize{ @work_cond_var.broadcast }
+      @worker_pool.shutdown(0)
+    end
+
+    should "know if and how many workers are available" do
+      wait_for_workers_to_become_available
+      assert_equal @num_workers, subject.available_worker_count
+      assert_true subject.worker_available?
+
+      # make one worker unavailable
+      subject.add_work(@work_item)
+
+      wait_for_a_worker_to_become_unavailable
+      assert_equal @num_workers - 1, subject.available_worker_count
+      assert_true subject.worker_available?
+
+      # make the rest of the workers unavailable
+      (@num_workers - 1).times{ subject.add_work(@work_item) }
+
+      wait_for_workers_to_become_unavailable
+      assert_equal 0, subject.available_worker_count
+      assert_false subject.worker_available?
+
+      # make one worker available
+      @work_mutex.synchronize{ @work_cond_var.signal }
+
+      wait_for_a_worker_to_become_available
+      assert_equal 1, subject.available_worker_count
+      assert_true subject.worker_available?
+
+      # make the rest of the workers available
+      @work_mutex.synchronize{ @work_cond_var.broadcast }
+
+      wait_for_workers_to_become_available
+      assert_equal @num_workers, subject.available_worker_count
+      assert_true subject.worker_available?
+    end
+
+  end
+
+  class WorkerCallbackTests < SystemTests
+    setup do
+      @worker_class = Class.new do
+        include SystemTestWorker
+
+        on_start{    params[:callbacks_called][:on_start]    = true }
+        on_shutdown{ params[:callbacks_called][:on_shutdown] = true }
+
+        on_available{   params[:callbacks_called][:on_available]   = true }
+        on_unavailable{ params[:callbacks_called][:on_unavailable] = true }
+
+        on_error{ |e, wi| params[:callbacks_called][:on_error] = true }
+
+        before_work{ |wi| params[:callbacks_called][:before_work] = true }
+        after_work{ |wi|  params[:callbacks_called][:after_work]  = true }
+
+        on_available{   signal_test_suite_thread }
+        on_unavailable{ signal_test_suite_thread }
+
+        def work!(work_item)
+          params[:finished].push(work_item)
+          signal_test_suite_thread
+          raise if work_item == 'error'
+        end
+      end
+      # use one worker to simplify; we only need to see that one worker runs its
+      # callbacks
+      @num_workers      = 1
+      @callbacks_called = {}
+      @finished         = LockedArray.new
+
+      @worker_pool = DatWorkerPool.new(@worker_class, {
+        :num_workers   => @num_workers,
+        :logger        => TEST_LOGGER,
+        :worker_params => @worker_params.merge({
+          :callbacks_called => @callbacks_called,
+          :finished         => @finished
+        })
+      })
+    end
+    teardown do
+      @worker_pool.shutdown(0)
+    end
+
+    should "run worker callbacks when started" do
+      assert_nil @callbacks_called[:on_start]
+      assert_nil @callbacks_called[:on_available]
+
+      subject.start
+      wait_for_workers_to_become_available
+
+      assert_true @callbacks_called[:on_start]
+      assert_true @callbacks_called[:on_available]
+    end
+
+    should "run worker callbacks when work is pushed" do
+      subject.start
+      wait_for_workers_to_become_available
+      @callbacks_called.delete(:on_available)
+
+      assert_nil @callbacks_called[:on_unavailable]
+      assert_nil @callbacks_called[:before_work]
+      assert_nil @callbacks_called[:after_work]
+
+      subject.add_work(Factory.string)
+      wait_for_workers do
+        @finished.size == @num_workers &&
+        subject.available_worker_count == @num_workers
+      end
+
+      assert_true @callbacks_called[:on_unavailable]
+      assert_true @callbacks_called[:before_work]
+      assert_true @callbacks_called[:after_work]
+      assert_true @callbacks_called[:on_available]
+    end
+
+    should "run worker callbacks when it errors" do
+      subject.start
+      wait_for_workers_to_become_available
+      @callbacks_called.delete(:on_available)
+
+      assert_nil @callbacks_called[:on_unavailable]
+      assert_nil @callbacks_called[:before_work]
+      assert_nil @callbacks_called[:on_error]
+      assert_nil @callbacks_called[:after_work]
+
+      subject.add_work('error')
+      wait_for_workers do
+        @finished.size == @num_workers &&
+        subject.available_worker_count == @num_workers
+      end
+
+      assert_true @callbacks_called[:on_unavailable]
+      assert_true @callbacks_called[:before_work]
+      assert_true @callbacks_called[:on_error]
+      assert_nil @callbacks_called[:after_work]
+      assert_true @callbacks_called[:on_available]
+    end
+
+    should "run callbacks when its shutdown" do
+      subject.start
+      wait_for_workers_to_become_available
+
+      assert_nil @callbacks_called[:on_unavailable]
+      assert_nil @callbacks_called[:on_shutdown]
+
+      subject.shutdown(0)
+
+      assert_true @callbacks_called[:on_unavailable]
+      assert_true @callbacks_called[:on_shutdown]
+    end
+
+  end
+
+  class ShutdownSystemTests < SystemTests
+    setup do
+      @worker_class = Class.new do
+        include SystemTestWorker
+        on_available{   signal_test_suite_thread }
+        on_unavailable{ signal_test_suite_thread }
+
+        on_error do |error, wi|
+          params[:errored].push([error, wi])
+        end
+
+        # this allows controlling how long a worker takes to finish processing
+        # the work item
+        def work!(work_item)
+          params[:work_mutex].synchronize do
+            params[:work_cond_var].wait(params[:work_mutex])
+          end
+          params[:finished].push(work_item)
+        end
+      end
+      @work_mutex    = Mutex.new
+      @work_cond_var = ConditionVariable.new
+      @finished      = LockedArray.new
+      @errored       = LockedArray.new
+
+      @worker_pool = DatWorkerPool.new(@worker_class, {
+        :num_workers   => @num_workers,
+        :logger        => TEST_LOGGER,
+        :worker_params => @worker_params.merge({
+          :work_mutex    => @work_mutex,
+          :work_cond_var => @work_cond_var,
+          :finished      => @finished,
+          :errored       => @errored
+        })
+      })
+
+      @worker_pool.start
+      wait_for_workers_to_become_available
+
+      # add 1 more work item than we have workers to handle it
+      @work_items = (@num_workers + 1).times.map{ Factory.string }
+      @work_items.each{ |wi| @worker_pool.add_work(wi) }
+      wait_for_workers_to_become_unavailable
+    end
+    teardown do
+      # ensure we wakeup any workers still stuck in their `work!`
+      @work_mutex.synchronize{ @work_cond_var.broadcast }
+      @worker_pool.shutdown(0)
+    end
+
+    should "allow any work that has been picked up to finish processing " \
+           "when shutdown without a timeout" do
+      assert_true @finished.empty?
+      assert_true @errored.empty?
+
+      # start the shutdown in a thread, this will hang it indefinitely because
+      # it has no timeout and the workers will never exit on their own (because
+      # they are waiting to be signaled by the cond var)
+      shutdown_thread = Thread.new{ subject.shutdown }
+      shutdown_thread.join(JOIN_SECONDS)
+      assert_equal 'sleep', shutdown_thread.status
+
+      # allow the workers to finish working
+      @work_mutex.synchronize{ @work_cond_var.broadcast }
+      wait_for_workers{ @finished.size == @num_workers }
+
+      # ensure we finished what we started processing; assert the size and
+      # includes because we can't ensure the order that the work items are
+      # finished
+      assert_equal @num_workers, @finished.size
+      @finished.values.each do |work_item|
+        assert_includes work_item, @work_items[0, @num_workers]
+      end
+
+      # ensure it didn't pick up anymore work
+      assert_equal [@work_items.last], subject.queue.work_items
+
+      # ensure nothing errored
+      assert_true @errored.empty?
+
+      # ensure the shutdown exits
+      shutdown_thread.join
+      assert_false shutdown_thread.alive?
+    end
+
+    should "allow any work that has been picked up to finish processing " \
+           "when forced to shutdown because it timed out" do
+      assert_true @finished.empty?
+      assert_true @errored.empty?
+
+      # start the shutdown in a thread, this will hang until the timeout
+      # finishes; this is required otherwise system timer will think we are
+      # triggering a deadlock (it's not a deadlock because of the timeout)
+      shutdown_thread = Thread.new{ subject.shutdown(0) }
+      shutdown_thread.join(JOIN_SECONDS)
+      assert_equal 'sleep', shutdown_thread.status
+
+      # wait for the workers to get forced to exit
+      wait_for_workers{ @errored.size == @num_workers }
+
+      # ensure it didn't finish what it started processing
+      assert_true @finished.empty?
+
+      # ensure it didn't pick up anymore work
+      assert_equal [@work_items.last], subject.queue.work_items
+
+      # ensure all the work it picked up was reported to its on-error callback
+      assert_equal @num_workers, @errored.size
+      @errored.values.each do |(exception, work_item)|
+        assert_instance_of ShutdownError, exception
+        assert_includes work_item, @work_items[0, @num_workers]
+      end
+
+      # ensure the shutdown exits
+      shutdown_thread.join
+      assert_false shutdown_thread.alive?
+    end
+
+  end
+
+  module SystemTestWorker
+    def self.included(klass)
+      klass.class_eval{ include DatWorkerPool::Worker }
+    end
+
+    def signal_test_suite_thread
+      params[:mutex].synchronize{ params[:cond_var].signal }
+    end
+  end
+
+end