ActionPool 0.2.1 → 0.2.2

data/CHANGELOG

@@ -1,3 +1,7 @@
+0.2.2
+    - Better thread management within the pool
+    - Restart thread timeouts when modified
+    - Fix argument passing
 0.2.1
     - Set minimum pool size properly when initialized to avoid
       creating extra threads (thanks to Roger Pack)

data/actionpool.gemspec

@@ -2,7 +2,7 @@ spec = Gem::Specification.new do |s|
     s.name              = 'ActionPool'
     s.author            = %q(spox)
     s.email             = %q(spox@modspox.com)
-    s.version           = '0.2.1'
+    s.version           = '0.2.2'
     s.summary           = %q(Thread Pool)
     s.platform          = Gem::Platform::RUBY
     s.files             = Dir['**/*']
@@ -11,13 +11,5 @@ spec = Gem::Specification.new do |s|
     s.require_paths     = %w(lib)
     s.required_ruby_version = '>= 1.8.6'
     s.homepage          = %q(http://github.com/spox/actionpool)
-    description         = []
-    File.open("README.rdoc") do |file|
-        file.each do |line|
-            line.chomp!
-            break if line.empty?
-            description << "#{line.gsub(/\[\d\]/, '')}"
-        end
-    end
-    s.description = description[1..-1].join(" ")
+    s.description         = "The ActionPool is an easy to use thread pool for ruby."
 end

data/lib/actionpool/Pool.rb

@@ -4,6 +4,9 @@ require 'actionpool/LogHelper'
 require 'thread'
 
 module ActionPool
+    # Raised when pool is closed
+    class PoolClosed < StandardError
+    end
     class Pool
 
         # :min_threads:: minimum number of threads in pool
@@ -23,35 +26,68 @@ module ActionPool
             @max_threads = args[:max_threads] ? args[:max_threads] : 100
             @min_threads = args[:min_threads] ? args[:min_threads] : 10
             @min_threads = @max_threads if @max_threads < @min_threads
-            @respond_to = ::Thread.current
-            create_thread
+            @respond_to = args[:respond_thread] || ::Thread.current
+            @open = true
+            fill_pool
         end
 
-        # force:: force creation of a new thread
-        # Create a new thread for pool. Returns newly created ActionPool::Thread or
-        # nil if pool has reached maximum threads
-        def create_thread(force=false)
-            pt = nil
+        # Pool is closed
+        def pool_closed?
+            !@open
+        end
+
+        # Pool is open
+        def pool_open?
+            @open
+        end
+
+        # arg:: :open or :closed
+        # Set pool status
+        def status(arg)
+            @open = arg == :open
+            fill_pool if @open
+        end
+
+        # args:: :force forces a new thread. :nowait will create a thread if threads are waiting
+        # Create a new thread for pool.
+        # Returns newly created thread of nil if pool is at maximum size
+        def create_thread(*args)
+            return if pool_closed?
+            thread = nil
             @lock.synchronize do
-                if(@threads.size < @max_threads || force)
-                    @logger.info('Pool is creating a new thread')
-                    (min - size > 0 ? min - size : 1).times do |i|
-                        pt = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
-                        @threads << pt
+                if(((size == working || args.include?(:nowait)) && @threads.size < @max_threads) || args.include?(:force))
+                    thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
+                    @threads << thread
+                end
+            end
+            return thread
+        end
+
+        # Fills the pool with the minimum number of threads
+        # Returns array of created threads
+        def fill_pool
+            threads = []
+            @lock.synchronize do
+                required = min - size
+                if(required > 0)
+                    required.times do
+                        thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
+                        @threads << thread
+                        threads << thread
                     end
-                else
-                    @logger.info('Pool is at maximum size. Not creating new thread')
                 end
             end
-            return pt
+            return threads
         end
 
         # force:: force immediate stop
         # Stop the pool
         def shutdown(force=false)
-            args = [:wait]
-            args += [:force] if force
+            status(:closed)
+            args = []
+            args.push(:force) if force
             @logger.info("Pool is now shutting down #{force ? 'using force' : ''}")
+            @queue.clear if force
             @queue.wait_empty
             while(t = @threads.pop) do
                 t.stop(*args)
@@ -67,7 +103,7 @@ module ActionPool
                     queue(action)
                 when Array
                     raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless action.size == 2 and action[0].is_a?(Proc) and action[1].is_a?(Array)
-                    queue(*action.flatten)
+                    queue(action[0], action[1])
                 else
                     raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
             end
@@ -77,15 +113,17 @@ module ActionPool
         # action:: proc to be executed
         # Add a new proc/lambda to be executed
         def queue(action, *args)
+            raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
             raise ArgumentError.new('Expecting block') unless action.is_a?(Proc)
             @queue << [action, args]
             ::Thread.pass
-            create_thread if @queue.num_waiting < 1 # only start a new thread if we need it
+            create_thread
         end
 
         # jobs:: Array of proc/lambdas
         # Will queue a list of jobs into the pool
         def add_jobs(jobs)
+            raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
             raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
             @queue.pause
             begin
@@ -95,13 +133,14 @@ module ActionPool
                         @queue << [job, []]
                     when Array
                         raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
-                        @queue << job
+                        @queue << [job.shift, job]
                     else
                         raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
                     end
                 end
             ensure
-                create_thread
+                num = jobs.size - @threads.select{|t|t.waiting?}.size
+                num.times{ create_thread(:nowait) } if num > 0
                 @queue.unpause
             end
             true
@@ -181,7 +220,7 @@ module ActionPool
         # Set maximum allowed time thead may idle in pool
         def thread_timeout=(t)
             t = t.to_f
-            raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
+            raise ArgumentError.new('Value must be greater than zero or nil') unless t >= 0
             @thread_timeout = t
             @threads.each{|thread|thread.thread_timeout = t}
             t
@@ -192,7 +231,7 @@ module ActionPool
         # on a given action
         def action_timeout=(t)
             t = t.to_f
-            raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
+            raise ArgumentError.new('Value must be greater than zero or nil') unless t >= 0
             @action_timeout = t
             @threads.each{|thread|thread.action_timeout = t}
             t
@@ -220,6 +259,11 @@ module ActionPool
             lock.synchronize{ guard.broadcast }
         end
 
+        # Returns current number of threads in the pool working
+        def working
+            @threads.find_all{|t|!t.waiting?}.size
+        end
+
         private
 
         # Resize the pool

data/lib/actionpool/Queue.rb

@@ -7,32 +7,47 @@ module ActionPool
         def initialize
             super
             @wait = false
-            @lock = Mutex.new
-            @guard = ConditionVariable.new
-            @elock = Mutex.new
-            @eguard = ConditionVariable.new
+            @pause_lock = Mutex.new
+            @empty_lock = Mutex.new
+            @pause_guard = ConditionVariable.new
+            @empty_guard = ConditionVariable.new
         end
         # Stop the queue from returning results to requesting
         # threads. Threads will wait for results until signalled
         def pause
-            @wait = true
+            @pause_lock.synchronize{@wait = true}
         end
         # Allow the queue to return results. Any threads waiting
         # will have results given to them.
         def unpause
-            @wait = false
-            @lock.synchronize{ @guard.broadcast }
+            @pause_lock.synchronize do
+                @wait = false
+                @pause_guard.broadcast
+            end
         end
         # Check if queue needs to wait before returning
         def pop
-            @lock.synchronize{ @guard.wait(@lock) } if @wait
+            @pause_lock.synchronize do
+                @pause_guard.wait(@pause_lock) if @wait
+            end
             o = super
-            @elock.synchronize{ @eguard.broadcast } if empty?
+            @empty_lock.synchronize do
+                @empty_guard.broadcast if empty?
+            end
             return o
         end
+        # Clear queue
+        def clear
+            super
+            @empty_lock.synchronize do
+                @empty_guard.broadcast
+            end
+        end
         # Park a thread here until queue is empty
         def wait_empty
-            @elock.synchronize{ @eguard.wait(@elock) } if size > 0
+            @empty_lock.synchronize do
+                @empty_guard.wait(@empty_lock) if size > 0
+            end
         end
     end
 end

data/lib/actionpool/Thread.rb

@@ -1,7 +1,12 @@
 require 'timeout'
 
 module ActionPool
-
+    # Exception class used for waking up a thread
+    class Wakeup < StandardError
+    end
+    # Raised within a thread when the timeout is changed
+    class Retimeout < StandardError
+    end
     class Thread
         # :pool:: pool thread is associated with
         # :t_timeout:: max time a thread is allowed to wait for action
@@ -19,6 +24,7 @@ module ActionPool
             @action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
             @kill = false
             @logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
+            @lock = Mutex.new
             @thread = ::Thread.new{ start_thread }
         end
 
@@ -27,29 +33,30 @@ module ActionPool
         # Stop the thread
         def stop(*args)
             @kill = true
-            @thread.kill if args.include?(:force) || waiting?
-            # killing the thread if it is waiting? is a simple race condition
-            # in that, if the thread has just finished a task, but hasn't
-            # dropped by the queue line yet, we will see it as not waiting. we
-            # can counter that by waiting just a little bit longer than the allowed
-            # time to work on an action, and kill the thread if it is still around
-            if(args.include?(:wait))
-                unless(@thread.join(@action_timeout + 0.01))
-                    @thread.kill
-                    @thread.join
-                end
-            end
+            @thread.raise Wakeup.new if args.include?(:force) || waiting?
             nil
         end
 
+        # Currently waiting
+        def waiting?
+            @lock.synchronize{@status} == :wait
+        end
+
         # Is the thread still alive
         def alive?
             @thread.alive?
         end
 
-        # Is the thread currently waiting for input
-        def waiting?
-            @thread.status == 'sleep'
+        # Current thread status
+        def status
+            @lock.synchronize{ return @status }
+        end
+        
+        # arg:: :wait or :run
+        # Set current status
+        def status(arg)
+            raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
+            @lock.synchronize{ @status = arg }
         end
 
         # Seconds thread will wait for input
@@ -68,11 +75,13 @@ module ActionPool
             t = t.to_f
             raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
             @thread_timeout = t
+            @thread.raise Retimeout.new if waiting?
             t
         end
 
         # t:: seconds to work on a task (floats allow for values 0 < t < 1)
         # Set the maximum amount of time to work on a given task
+        # Note: Modification of this will not affect actions already in process
         def action_timeout=(t)
             t = t.to_f
             raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
@@ -87,29 +96,42 @@ module ActionPool
             begin
                 @logger.info("New pool thread is starting (#{self})")
                 until(@kill) do
+                    status(:wait)
                     begin
                         action = nil
-                        if(@pool.size > @pool.min)
+                        if(@pool.size > @pool.min && !@thread_timeout.zero?)
                             Timeout::timeout(@thread_timeout) do
                                 action = @pool.action
                             end
                         else
                             action = @pool.action
                         end
-                        run(*action) unless action.nil?
+                        status(:run)
+                        run(action[0], action[1]) unless action.nil?
+                        status(:wait)
                     rescue Timeout::Error => boom
                         @kill = true
+                    rescue Wakeup
+                        @logger.info("Thread #{::Thread.current} was woken up.")
+                    rescue Retimeout
+                        @logger.warn('Thread was woken up to reset thread timeout')
                     rescue Exception => boom
                         @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
                         @respond_to.raise boom
                     end
                 end
+            rescue Retimeout
+                @logger.warn('Thread was woken up to reset thread timeout')
+                retry
+            rescue Wakeup
+                @logger.info("Thread #{::Thread.current} was woken up.")
             rescue Exception => boom
                 @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
                 @respond_to.raise boom
             ensure
                 @logger.info("Pool thread is shutting down (#{self})")
                 @pool.remove(self)
+                @pool.create_thread
             end
         end
 
@@ -118,12 +140,12 @@ module ActionPool
         # Run the task
         def run(action, args)
             begin
-                if(@action_timeout > 0)
+                unless(@action_timeout.zero?)
                     Timeout::timeout(@action_timeout) do
-                        action.call(*args)
+                        action.call(*args[0])
                     end
                 else
-                    action.call(*args)
+                    action.call(*args[0])
                 end
             rescue Timeout::Error => boom
                 @logger.warn("Pool thread reached max execution time for action: #{boom}")

data/tests/cases/general.rb

@@ -5,6 +5,9 @@ class GeneralPoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
+    def teardown
+        @pool.shutdown(true)
+    end
     def test_numbers
         assert_equal(10, @pool.size)
         assert_equal(10, @pool.min)
@@ -20,6 +23,7 @@ class GeneralPoolTest < Test::Unit::TestCase
         100.times{ @pool << run }
         @pool.shutdown
         assert_equal(100, a)
+        @pool.status :open
         a = 0
         jobs = [].fill(run,0,100)
         @pool.add_jobs(jobs)
@@ -28,15 +32,21 @@ class GeneralPoolTest < Test::Unit::TestCase
         @pool.shutdown(true)
     end
     def test_args
+        @pool.status :open
         output = nil
         @pool << [lambda{|x| output = x}, [2]]
         assert(2, output)
         @pool.add_jobs([[lambda{|x| output = x}, [3]]])
         assert(3, output)
+        @pool << [lambda{|x,y| output = x+y}, [1,2]]
+        assert(3, output)
         output = []
         @pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
         assert(output.include?(2))
         assert(output.include?(3))
+        @pool << [lambda{|x,y| output = [x,y]}, ['test', [1,2]]]
+        assert_equal(output[0], 'test')
+        assert(output[1].is_a?(Array))
         @pool.shutdown(true)
     end
 end

data/tests/cases/grow.rb

@@ -5,16 +5,20 @@ class GrowPoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
+    def teardown
+        @pool.shutdown(true)
+    end
     def test_grow
-        jobs = [].fill(lambda{}, 0..99)
+        jobs = [].fill(lambda{sleep}, 0..20)
         @pool.add_jobs(jobs)
+        Thread.pass
         assert(@pool.size > 10)
         @pool.shutdown(true)
     end
     def test_max
-        @pool.create_thread(true) until @pool.size > @pool.max
-        assert_nil(@pool.create_thread)
-        assert(@pool.create_thread(true))
+        @pool.create_thread(:force) until @pool.size > @pool.max
+        assert(@pool.create_thread.nil?)
+        assert(!@pool.create_thread(:force).nil?)
         @pool.shutdown(true)
     end
 end

data/tests/cases/nogrow.rb

@@ -5,6 +5,9 @@ class NoGrowPoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
+    def teardown
+        @pool.shutdown(true)
+    end
     def test_nogrow
         5.times{ @pool << lambda{} }
         assert_equal(10, @pool.size)

data/tests/cases/resize.rb

@@ -5,20 +5,26 @@ class ResizePoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
+    def teardown
+        @pool.shutdown(true)
+    end
     def test_resize
         stop = false
-        20.times{ @pool << lambda{ a = 0; a += 1 until stop || a > 9999999999 } }
+        20.times{ @pool << lambda{ sleep } }
+        Thread.pass
+        sleep(0.1)
         assert(@pool.size > 10)
         stop = true
         @pool.shutdown(true)
+        @pool.status :open
         @pool.max = 10
         assert_equal(10, @pool.max)
-        assert_equal(0, @pool.size)
         stop = false
         20.times{ @pool << lambda{ a = 0; a += 1 until stop || a > 9999999999 } }
         assert_equal(10, @pool.size)
         stop = true
         @pool.shutdown(true)
+        @pool.status :open
         @pool.max = 20
         stop = false
         30.times{ @pool << lambda{ a = 0; a += 1 until stop || a > 9999999999 } }

data/tests/cases/shutdown.rb

@@ -5,12 +5,32 @@ class ShutdownPoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
+    def teardown
+        @pool.shutdown(true)
+    end
+    def test_close
+        result = 0
+        @pool << lambda{ result = 5 }
+        sleep(0.01)
+        assert(5, result)
+        @pool.status :closed
+        assert_raise(ActionPool::PoolClosed) do
+            @pool << lambda{}
+        end
+        assert_raise(ActionPool::PoolClosed) do
+            @pool.add_jobs [lambda{}, lambda{}]
+        end
+        @pool.shutdown(true)
+    end
     def test_shutdown
         assert_equal(10, @pool.size)
         @pool.shutdown
         sleep(0.5)
         assert_equal(0, @pool.size)
-        @pool << lambda{}
+        @pool.status :open
+        @pool << lambda{ sleep }
+        sleep(0.01)
         assert_equal(10, @pool.size)
+        @pool.shutdown(true)
     end
 end

data/tests/cases/thread.rb

@@ -6,6 +6,9 @@ class ThreadTest < Test::Unit::TestCase
         @pool = ActionPool::Pool.new(:min_threads => 1, :max_threads => 1)
         @thread = ActionPool::Thread.new(:pool => @pool, :respond_thread => self, :t_timeout => 60, :a_timeout => 0)
     end
+    def teardown
+        @pool.shutdown(true)
+    end
     def test_thread
         sleep(0.01)
         assert(@thread.waiting?)

data/tests/cases/timeouts.rb

@@ -5,15 +5,33 @@ class TimeoutPoolTest < Test::Unit::TestCase
     def setup
         @pool = ActionPool::Pool.new
     end
-    def test_threadtimeout
-        @pool.thread_timeout = 0.01
+    def teardown
+        @pool.shutdown(true)
+    end
+    def test_actiontimeout
+        @pool.action_timeout = 0.01
         assert_equal(10, @pool.size)
         stop = false
-        20.times{ @pool << lambda{ a = 0; a += 1 until stop || a > 9999999999 } }
-        assert(@pool.size > 10)
+        @pool.add_jobs [].fill(lambda{loop{ 1+1 }}, 0, 20)
+        ::Thread.pass
+        assert(@pool.working > 10)
         stop = true
-        sleep(2)
-        assert(@pool.size <= 10)
+        sleep(0.5)
+        assert(@pool.working == 0)
+        @pool.shutdown(true)
+    end
+    def test_threadtimeout
+        @pool.thread_timeout = 0.01
+        assert_equal(10, @pool.size)
+        lock = Mutex.new
+        guard = ConditionVariable.new
+        @pool.add_jobs [].fill(lambda{ lock.synchronize{ guard.wait(lock) } }, 0, 20)
+        ::Thread.pass
+        assert_equal(30, @pool.size)
+        lock.synchronize{ guard.broadcast }
+        ::Thread.pass
+        sleep(0.1)
+        assert(10, @pool.size)
         @pool.shutdown(true)
     end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ActionPool
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors: 
 - spox
@@ -9,11 +9,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-04 00:00:00 -08:00
+date: 2009-12-11 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
-description: ""
+description: The ActionPool is an easy to use thread pool for ruby.
 email: spox@modspox.com
 executables: []
 
@@ -25,8 +25,6 @@ extra_rdoc_files:
 files: 
 - Rakefile
 - actionpool.gemspec
-- tests
-- tests/cases
 - tests/cases/general.rb
 - tests/cases/resize.rb
 - tests/cases/nogrow.rb
@@ -36,8 +34,6 @@ files:
 - tests/cases/queue.rb
 - tests/cases/grow.rb
 - tests/run_tests.rb
-- lib
-- lib/actionpool
 - lib/actionpool/Pool.rb
 - lib/actionpool/Thread.rb
 - lib/actionpool/LogHelper.rb
@@ -46,8 +42,10 @@ files:
 - CHANGELOG
 - LICENSE
 - README.rdoc
-has_rdoc: false
+has_rdoc: true
 homepage: http://github.com/spox/actionpool
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --title
@@ -72,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Thread Pool
 test_files: []