dat-worker-pool 0.1.0 → 0.2.0

data/dat-worker-pool.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
   gem.description = "A simple thread pool for processing generic 'work'"
   gem.summary     = "A simple thread pool for processing generic 'work'"
   gem.homepage    = "http://github.com/redding/dat-worker-pool"
+  gem.license     = 'MIT'
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/dat-worker-pool.rb

@@ -37,6 +37,22 @@ class DatWorkerPool
     @workers_waiting.count
   end
 
+  def worker_available?
+    !reached_max_workers? || @workers_waiting.count > 0
+  end
+
+  def all_spawned_workers_are_busy?
+    @workers_waiting.count <= 0
+  end
+
+  def reached_max_workers?
+    @mutex.synchronize{ @spawned >= @max_workers }
+  end
+
+  def queue_empty?
+    @queue.empty?
+  end
+
   # Check if all workers are busy before adding the work. When the work is
   # added, a worker will stop waiting (if it was idle). Because of that, we
   # can't reliably check if all workers are busy. We might think all workers are
@@ -44,9 +60,9 @@ class DatWorkerPool
   # would spawn a worker to do nothing.
   def add_work(work_item)
     return if work_item.nil?
-    new_worker_needed = all_workers_are_busy?
+    new_worker_needed = all_spawned_workers_are_busy?
     @queue.push work_item
-    self.spawn_worker if new_worker_needed && havent_reached_max_workers?
+    self.spawn_worker if new_worker_needed && !reached_max_workers?
   end
 
   # Shutdown each worker and then the queue. Shutting down the queue will
@@ -55,9 +71,9 @@ class DatWorkerPool
   # finish.
   # **NOTE** Any work that is left on the queue isn't processed. The controlling
   # application for the worker pool should gracefully handle these items.
-  def shutdown(timeout)
+  def shutdown(timeout = nil)
     begin
-      SystemTimer.timeout(timeout, TimeoutError) do
+      proc = OptionalTimeoutProc.new(timeout, true) do
         @workers.each(&:shutdown)
         @queue.shutdown
 
@@ -68,51 +84,44 @@ class DatWorkerPool
         # `each`).
         @workers.first.join until @workers.empty?
       end
+      proc.call
     rescue TimeoutError => exception
       exception.message.replace "Timed out shutting down the worker pool"
       @debug ? raise(exception) : self.logger.error(exception.message)
     end
   end
 
-  # public, because workers need to call it for themselves
-  def despawn_worker(worker)
-    @mutex.synchronize do
-      @spawned -= 1
-      @workers.delete worker
-    end
-  end
-
   protected
 
   def spawn_worker
     @mutex.synchronize do
-      worker = Worker.new(self, @queue, @workers_waiting) do |work_item|
-        do_work(work_item)
+      Worker.new(@queue).tap do |w|
+        w.on_work       = proc{ |work_item| do_work(work_item) }
+        w.on_waiting    = proc{ @workers_waiting.increment }
+        w.on_continuing = proc{ @workers_waiting.decrement }
+        w.on_shutdown   = proc{ |worker| despawn_worker(worker) }
+
+        @workers << w
+        @spawned += 1
+
+        w.start
       end
-      @workers << worker
-      @spawned += 1
-      worker
     end
   end
 
-  def do_work(work_item)
-    begin
-      @do_work_proc.call(work_item)
-    rescue Exception => exception
-      self.logger.error "Exception raised while doing work!"
-      self.logger.error "#{exception.class}: #{exception.message}"
-      self.logger.error exception.backtrace.join("\n")
+  def despawn_worker(worker)
+    @mutex.synchronize do
+      @spawned -= 1
+      @workers.delete worker
     end
   end
 
-  def all_workers_are_busy?
-    @workers_waiting.count <= 0
-  end
-
-  def havent_reached_max_workers?
-    @mutex.synchronize do
-      @spawned < @max_workers
-    end
+  def do_work(work_item)
+    @do_work_proc.call(work_item)
+  rescue Exception => exception
+    self.logger.error "Exception raised while doing work!"
+    self.logger.error "#{exception.class}: #{exception.message}"
+    self.logger.error exception.backtrace.join("\n")
   end
 
   class WorkersWaiting
@@ -133,6 +142,26 @@ class DatWorkerPool
 
   end
 
+  class OptionalTimeoutProc
+    def initialize(timeout, reraise = false, &proc)
+      @timeout = timeout
+      @reraise = reraise
+      @proc    = proc
+    end
+
+    def call
+      if @timeout
+        begin
+          SystemTimer.timeout(@timeout, TimeoutError, &@proc)
+        rescue TimeoutError
+          raise if @reraise
+        end
+      else
+        @proc.call
+      end
+    end
+  end
+
   module Logger
     def self.new(debug)
       debug ? ::Logger.new(STDOUT) : ::Logger.new(File.open('/dev/null', 'w'))

data/lib/dat-worker-pool/queue.rb

@@ -26,19 +26,17 @@ class DatWorkerPool
     end
 
     def pop
-      @mutex.synchronize{ @work_items.shift }
+      return if @shutdown
+      @mutex.synchronize do
+        @condition_variable.wait(@mutex) while !@shutdown && @work_items.empty?
+        @work_items.shift
+      end
     end
 
     def empty?
       @mutex.synchronize{ @work_items.empty? }
     end
 
-    # wait to be signaled by `push`
-    def wait_for_work_item
-      return if @shutdown
-      @mutex.synchronize{ @condition_variable.wait(@mutex) }
-    end
-
     # wake up any workers who are idle (because of `wait_for_work_item`)
     def shutdown
       @shutdown = true

data/lib/dat-worker-pool/version.rb

@@ -1,3 +1,3 @@
 class DatWorkerPool
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/dat-worker-pool/worker.rb

@@ -3,14 +3,25 @@ require 'thread'
 class DatWorkerPool
 
   class Worker
+    attr_writer :on_work, :on_waiting, :on_continuing, :on_shutdown
 
-    def initialize(pool, queue, workers_waiting, &block)
-      @pool            = pool
-      @queue           = queue
-      @workers_waiting = workers_waiting
-      @block           = block
-      @shutdown        = false
-      @thread          = Thread.new{ work_loop }
+    def initialize(queue)
+      @queue = queue
+      @on_work       = proc{ |work_item| }
+      @on_waiting    = proc{ |worker| }
+      @on_continuing = proc{ |worker| }
+      @on_shutdown   = proc{ |worker| }
+
+      @shutdown = false
+      @thread   = nil
+    end
+
+    def start
+      @thread ||= Thread.new{ work_loop }
+    end
+
+    def running?
+      @thread && @thread.alive?
     end
 
     def shutdown
@@ -18,29 +29,22 @@ class DatWorkerPool
     end
 
     def join(*args)
-      @thread.join(*args) if @thread
+      @thread.join(*args) if running?
     end
 
     protected
 
     def work_loop
       loop do
-        self.wait_for_work
+        @on_waiting.call(self)
+        work_item = @queue.pop
+        @on_continuing.call(self)
         break if @shutdown
-        @block.call @queue.pop
+        @on_work.call(work_item) if work_item
       end
     ensure
-      @pool.despawn_worker(self)
-    end
-
-    # Wait for work to process by checking if the queue is empty.
-    def wait_for_work
-      while @queue.empty?
-        return if @shutdown
-        @workers_waiting.increment
-        @queue.wait_for_work_item
-        @workers_waiting.decrement
-      end
+      @on_shutdown.call(self)
+      @thread = nil
     end
 
   end

data/test/unit/dat-worker-pool_tests.rb

@@ -3,7 +3,7 @@ require 'dat-worker-pool'
 
 class DatWorkerPool
 
-  class BaseTests < Assert::Context
+  class UnitTests < Assert::Context
     desc "DatWorkerPool"
     setup do
       @work_pool = DatWorkerPool.new{ }
@@ -13,34 +13,49 @@ class DatWorkerPool
     should have_readers :logger, :spawned
     should have_imeths :add_work, :shutdown, :despawn_worker
     should have_imeths :work_items, :waiting
+    should have_imeths :worker_available?, :all_spawned_workers_are_busy?
+    should have_imeths :reached_max_workers?
+    should have_imeths :queue_empty?
 
   end
 
-  class WorkerBehaviorTests < BaseTests
+  class WorkerBehaviorTests < UnitTests
     desc "workers"
     setup do
-      @work_pool = DatWorkerPool.new(1, 2, true){|work| sleep(work) }
+      @work_pool = DatWorkerPool.new(1, 2, true){ |work| sleep(work) }
     end
 
     should "be created as needed and only go up to the maximum number allowed" do
       # the minimum should be spawned and waiting
       assert_equal 1, @work_pool.spawned
       assert_equal 1, @work_pool.waiting
+      assert_equal true,  @work_pool.worker_available?
+      assert_equal false, @work_pool.all_spawned_workers_are_busy?
+      assert_equal false, @work_pool.reached_max_workers?
 
       # the minimum should be spawned, but no longer waiting
       @work_pool.add_work 5
       assert_equal 1, @work_pool.spawned
       assert_equal 0, @work_pool.waiting
+      assert_equal true,  @work_pool.worker_available?
+      assert_equal true,  @work_pool.all_spawned_workers_are_busy?
+      assert_equal false, @work_pool.reached_max_workers?
 
       # an additional worker should be spawned
       @work_pool.add_work 5
       assert_equal 2, @work_pool.spawned
       assert_equal 0, @work_pool.waiting
+      assert_equal false, @work_pool.worker_available?
+      assert_equal true,  @work_pool.all_spawned_workers_are_busy?
+      assert_equal true,  @work_pool.reached_max_workers?
 
       # no additional workers are spawned, the work waits to be processed
       @work_pool.add_work 5
       assert_equal 2, @work_pool.spawned
       assert_equal 0, @work_pool.waiting
+      assert_equal false, @work_pool.worker_available?
+      assert_equal true,  @work_pool.all_spawned_workers_are_busy?
+      assert_equal true, @work_pool.reached_max_workers?
     end
 
     should "go back to waiting when they finish working" do
@@ -59,7 +74,20 @@ class DatWorkerPool
 
   end
 
-  class AddWorkAndProcessItTests < BaseTests
+  class AddWorkWithNoWorkersTests < UnitTests
+    setup do
+      @work_pool = DatWorkerPool.new(0, 0){ |work| }
+    end
+
+    should "return whether or not the queue is empty" do
+      assert_equal true, @work_pool.queue_empty?
+      @work_pool.add_work 'test'
+      assert_equal false, @work_pool.queue_empty?
+    end
+
+  end
+
+  class AddWorkAndProcessItTests < UnitTests
     desc "add_work and process"
     setup do
       @result = nil
@@ -84,7 +112,7 @@ class DatWorkerPool
 
   end
 
-  class ShutdownTests < BaseTests
+  class ShutdownTests < UnitTests
     desc "shutdown"
     setup do
       @mutex = Mutex.new
@@ -101,7 +129,6 @@ class DatWorkerPool
     should "allow any work that has been picked up to be processed" do
       # make sure the workers haven't processed any work
       assert_equal [], @finished
-
       subject.shutdown(5)
 
       # NOTE, the last work shouldn't have been processed, as it wasn't
@@ -123,6 +150,13 @@ class DatWorkerPool
       end
     end
 
+    should "allow jobs to finish by not providing a shutdown timeout" do
+      assert_equal [], @finished
+      subject.shutdown
+      assert_includes 'a', @finished
+      assert_includes 'b', @finished
+    end
+
   end
 
 end

data/test/unit/queue_tests.rb

@@ -3,7 +3,7 @@ require 'dat-worker-pool/queue'
 
 class DatWorkerPool::Queue
 
-  class BaseTests < Assert::Context
+  class UnitTests < Assert::Context
     desc "DatWorkerPool::Queue"
     setup do
       @queue = DatWorkerPool::Queue.new
@@ -11,7 +11,7 @@ class DatWorkerPool::Queue
     subject{ @queue }
 
     should have_imeths :work_items, :push, :pop, :empty?
-    should have_imeths :wait_for_work_item, :shutdown
+    should have_imeths :shutdown
 
     should "allow pushing work items onto the queue with #push" do
       subject.push 'work'
@@ -34,6 +34,12 @@ class DatWorkerPool::Queue
       assert_equal 0, subject.work_items.size
     end
 
+    should "return nothing with pop when the queue has been shutdown" do
+      subject.push 'work1'
+      subject.shutdown
+      assert_nil subject.pop
+    end
+
     should "return whether the queue is empty or not with #empty?" do
       assert subject.empty?
       subject.push 'work'
@@ -42,25 +48,32 @@ class DatWorkerPool::Queue
       assert subject.empty?
     end
 
-    should "sleep a thread with #wait_for_work_item and " \
-           "wake it up with #push" do
-      thread = Thread.new{ subject.wait_for_work_item }
-      thread.join(0.1) # ensure the thread runs enough to start waiting
-      subject.push 'work'
-      # if this returns nil, then the thread never finished
-      assert_not_nil thread.join(1)
+  end
+
+  class SignallingTests < UnitTests
+    desc "mutex and condition variable behavior"
+    setup do
+      @thread = Thread.new do
+        Thread.current['work_item'] = @queue.pop || 'got nothing'
+      end
+    end
+
+    should "have threads wait for a work item to be added when using pop" do
+      assert_equal "sleep", @thread.status
+    end
+
+    should "wakeup threads when work is pushed onto the queue" do
+      subject.push 'some work'
+      sleep 0.1
+      assert !@thread.alive?
+      assert_equal 'some work', @thread['work_item']
     end
 
-    should "sleep threads with #wait_for_work_item and " \
-           "wake them all up with #shutdown" do
-      thread1 = Thread.new{ subject.wait_for_work_item }
-      thread1.join(0.1) # ensure the thread runs enough to start waiting
-      thread2 = Thread.new{ subject.wait_for_work_item }
-      thread2.join(0.1) # ensure the thread runs enough to start waiting
+    should "wakeup thread when the queue is shutdown" do
       subject.shutdown
-      # if these returns nil, then the threads never finished
-      assert_not_nil thread1.join(1)
-      assert_not_nil thread2.join(1)
+      sleep 0.1
+      assert !@thread.alive?
+      assert_equal 'got nothing', @thread['work_item']
     end
 
   end

data/test/unit/worker_tests.rb

@@ -6,24 +6,108 @@ require 'dat-worker-pool/queue'
 
 class DatWorkerPool::Worker
 
-  class BaseTests < Assert::Context
+  class UnitTests < Assert::Context
     desc "DatWorkerPool::Worker"
     setup do
-      @pool  = DatWorkerPool.new{ }
       @queue = DatWorkerPool::Queue.new
-      @workers_waiting = DatWorkerPool::WorkersWaiting.new
-      @worker = DatWorkerPool::Worker.new(@pool, @queue, @workers_waiting){ }
+      @work_done = []
+      @worker = DatWorkerPool::Worker.new(@queue).tap do |w|
+        w.on_work = proc{ |work| @work_done << work }
+      end
+    end
+    teardown do
+      @worker.shutdown
+      @queue.shutdown
+      @worker.join
     end
     subject{ @worker }
 
-    should have_imeths :shutdown, :join
+    should have_writers :on_waiting, :on_continuing, :on_shutdown
+    should have_imeths :start, :shutdown, :join, :running?
+
+    should "start a thread with it's work loop with #start" do
+      thread = nil
+      assert_nothing_raised{ thread = subject.start }
+      assert_instance_of Thread, thread
+      assert thread.alive?
+      assert subject.running?
+    end
+
+    should "call the block it's passed when it get's work from the queue" do
+      subject.start
+      @queue.push 'one'
+      subject.join 0.1 # trigger the worker's thread to run
+      @queue.push 'two'
+      subject.join 0.1 # trigger the worker's thread to run
+      assert_equal [ 'one', 'two' ], @work_done
+    end
+
+    should "flag itself for exiting it's work loop with #shutdown and " \
+           "end it's thread once it's queue is shutdown" do
+      thread = subject.start
+      subject.join 0.1 # trigger the worker's thread to run, allow it to get into it's
+                       # work loop
+      assert_nothing_raised{ subject.shutdown }
+      @queue.shutdown
+
+      subject.join 0.1 # trigger the worker's thread to run, should exit
+      assert_not thread.alive?
+      assert_not subject.running?
+    end
+
+  end
+
+  class CallbacksTests < UnitTests
+    desc "callbacks"
+    setup do
+      @worker = DatWorkerPool::Worker.new(@queue).tap do |w|
+        w.on_work = proc{ |work| sleep 0.2 }
+      end
+    end
+
+    should "call the on waiting callback, yielding itself, when " \
+           "it's waiting on work from the queue" do
+      waiting, yielded_worker = nil, nil
+      subject.on_waiting = proc do |worker|
+        waiting = true
+        yielded_worker = worker
+      end
+      subject.start
+      subject.join 0.1 # trigger the worker's thread to run
+
+      assert_equal true, waiting
+      assert_equal subject, yielded_worker
+    end
+
+    should "call the on continuing callback, yielding itself, when " \
+           "it's done waiting for work from the queue" do
+      waiting, yielded_worker = nil, nil
+      subject.on_continuing = proc do |worker|
+        waiting = false
+        yielded_worker = worker
+      end
+      subject.start
+      @queue.push 'some work'
+      subject.join 0.1 # trigger the worker's thread to run
 
-    should "trigger exiting it's work loop with #shutdown and " \
-           "join it's thread with #join" do
-      @queue.shutdown   # ensure the thread is not waiting on the queue
-      subject.join(0.1) # ensure the thread is looping for work
+      assert_equal false, waiting
+      assert_equal subject, yielded_worker
+    end
+
+    should "call the on shutdown callback, yielding itself, when " \
+           "it's shutdown" do
+      shutdown, yielded_worker = nil, nil
+      subject.on_shutdown = proc do |worker|
+        shutdown = true
+        yielded_worker = worker
+      end
+      subject.start
       subject.shutdown
-      assert_not_nil subject.join(1)
+      @queue.shutdown
+      subject.join 0.1 # trigger the worker's thread to run
+
+      assert_equal true, shutdown
+      assert_equal subject, yielded_worker
     end
 
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dat-worker-pool
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Collin Redding
@@ -16,10 +16,9 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-07-15 00:00:00 Z
+date: 2013-10-01 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  prerelease: false
   version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -30,11 +29,11 @@ dependencies:
         - 1
         - 2
         version: "1.2"
+  type: :runtime
   requirement: *id001
+  prerelease: false
   name: SystemTimer
-  type: :runtime
 - !ruby/object:Gem::Dependency 
-  prerelease: false
   version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -44,9 +43,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  type: :development
   requirement: *id002
+  prerelease: false
   name: assert
-  type: :development
 description: A simple thread pool for processing generic 'work'
 email: 
 - collin.redding@me.com
@@ -76,8 +76,8 @@ files:
 - test/unit/worker_tests.rb
 - tmp/.gitkeep
 homepage: http://github.com/redding/dat-worker-pool
-licenses: []
-
+licenses: 
+- MIT
 post_install_message: 
 rdoc_options: []