actionpool 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG CHANGED
@@ -1,3 +1,9 @@
1
+ 0.2.3
2
+ - Better thread destroying when removed normally and forcibly
3
+ - Flushing threads to ensure proper joining
4
+ - Fixed splatting for ruby < 1.9
5
+ - Removed the useless logger wrapper
6
+ - Use a simple monitor
1
7
  0.2.2
2
8
  - Better thread management within the pool
3
9
  - Restart thread timeouts when modified
@@ -4,7 +4,7 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
4
4
 
5
5
  === install (easy):
6
6
 
7
- gem install ActionPool
7
+ gem install actionpool
8
8
 
9
9
  === install (less easy):
10
10
 
@@ -17,6 +17,15 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
17
17
 
18
18
  {rip}[http://hellorip.com/about.html] makes it easy to install directly from a github repository.
19
19
 
20
+ === Testing
21
+
22
+ ActionPool is currently tested on:
23
+
24
+ * Ruby 1.8.6-p383
25
+ * Ruby 1.8.7-p248
26
+ * Ruby 1.9.1-p376
27
+ * JRuby 1.4.0
28
+
20
29
  == Documentation
21
30
 
22
31
  {rdocs}[http://allgems.ruby-forum.com/gems/ActionPool/]
@@ -2,7 +2,7 @@ spec = Gem::Specification.new do |s|
2
2
  s.name = 'actionpool'
3
3
  s.author = %q(spox)
4
4
  s.email = %q(spox@modspox.com)
5
- s.version = '0.2.2'
5
+ s.version = '0.2.3'
6
6
  s.summary = %q(Thread Pool)
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.files = Dir['**/*']
@@ -10,6 +10,7 @@ spec = Gem::Specification.new do |s|
10
10
  s.extra_rdoc_files = %w(README.rdoc CHANGELOG)
11
11
  s.require_paths = %w(lib)
12
12
  s.required_ruby_version = '>= 1.8.6'
13
+ s.add_dependency 'splib', '~> 1.4'
13
14
  s.homepage = %q(http://github.com/spox/actionpool)
14
15
  s.description = "The ActionPool is an easy to use thread pool for ruby."
15
16
  end
@@ -1,7 +1,10 @@
1
+ require 'rubygems'
1
2
  begin
2
3
  require 'fastthread'
3
4
  rescue LoadError
4
5
  # we don't care if it's available
5
6
  # just load it if it's around
6
7
  end
8
+ require 'splib'
9
+ Splib.load :Array, :Monitor
7
10
  require 'actionpool/Pool'
@@ -1,6 +1,6 @@
1
1
  require 'actionpool/Thread'
2
2
  require 'actionpool/Queue'
3
- require 'actionpool/LogHelper'
3
+ require 'logger'
4
4
  require 'thread'
5
5
 
6
6
  module ActionPool
@@ -17,10 +17,10 @@ module ActionPool
17
17
  # Creates a new pool
18
18
  def initialize(args={})
19
19
  raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
20
- @logger = LogHelper.new(args[:logger])
20
+ @logger = args[:logger] && args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
21
21
  @queue = ActionPool::Queue.new
22
22
  @threads = []
23
- @lock = Mutex.new
23
+ @lock = Splib::Monitor.new
24
24
  @thread_timeout = args[:t_to] ? args[:t_to] : 0
25
25
  @action_timeout = args[:a_to] ? args[:a_to] : 0
26
26
  @max_threads = args[:max_threads] ? args[:max_threads] : 100
@@ -48,36 +48,44 @@ module ActionPool
48
48
  fill_pool if @open
49
49
  end
50
50
 
51
- # args:: :force forces a new thread. :nowait will create a thread if threads are waiting
51
+ # args:: :force forces a new thread.
52
+ # :nowait will create a thread if threads are waiting
52
53
  # Create a new thread for pool.
53
- # Returns newly created thread of nil if pool is at maximum size
54
+ # Returns newly created thread or nil if pool is at maximum size
54
55
  def create_thread(*args)
55
56
  return if pool_closed?
56
57
  thread = nil
57
58
  @lock.synchronize do
58
59
  if(((size == working || args.include?(:nowait)) && @threads.size < @max_threads) || args.include?(:force))
59
- thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
60
+ thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
61
+ :t_timeout => @thread_timeout, :logger => @logger, :autostart => false)
60
62
  @threads << thread
61
63
  end
62
64
  end
63
- return thread
65
+ thread.start if thread
66
+ thread
64
67
  end
65
68
 
66
69
  # Fills the pool with the minimum number of threads
67
70
  # Returns array of created threads
68
71
  def fill_pool
69
72
  threads = []
70
- @lock.synchronize do
71
- required = min - size
72
- if(required > 0)
73
- required.times do
74
- thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
75
- @threads << thread
76
- threads << thread
73
+ if(@open)
74
+ @lock.synchronize do
75
+ required = min - size
76
+ if(required > 0)
77
+ required.times do
78
+ thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to,
79
+ :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger,
80
+ :autostart => false)
81
+ @threads << thread
82
+ threads << thread
83
+ end
77
84
  end
78
85
  end
79
86
  end
80
- return threads
87
+ threads.each{|t|t.start}
88
+ threads
81
89
  end
82
90
 
83
91
  # force:: force immediate stop
@@ -92,6 +100,10 @@ module ActionPool
92
100
  while(t = @threads.pop) do
93
101
  t.stop(*args)
94
102
  end
103
+ unless(force)
104
+ flush
105
+ @threads.each{|t|t.join}
106
+ end
95
107
  nil
96
108
  end
97
109
 
@@ -193,12 +205,10 @@ module ActionPool
193
205
  def remove(t)
194
206
  raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
195
207
  t.stop
196
- if(@threads.include?(t))
197
- @threads.delete(t)
198
- return true
199
- else
200
- return false
201
- end
208
+ del = @threads.include?(t)
209
+ @threads.delete(t) if del
210
+ fill_pool
211
+ del
202
212
  end
203
213
 
204
214
  # Maximum number of seconds a thread
@@ -251,17 +261,20 @@ module ActionPool
251
261
  # the pool if existing threads have a long thread life waiting
252
262
  # for input.
253
263
  def flush
254
- lock = Mutex.new
255
- guard = ConditionVariable.new
256
- @threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
257
- Thread.pass
264
+ mon = Splib::Monitor.new
265
+ @threads.size.times{ queue{ mon.wait } }
266
+ @queue.wait_empty
258
267
  sleep(0.01)
259
- lock.synchronize{ guard.broadcast }
268
+ mon.broadcast
260
269
  end
261
270
 
262
271
  # Returns current number of threads in the pool working
263
272
  def working
264
- @threads.find_all{|t|!t.waiting?}.size
273
+ @threads.select{|t|t.running?}.size
274
+ end
275
+
276
+ def thread_stats
277
+ @threads.map{|t|[t.object_id,t.status]}
265
278
  end
266
279
 
267
280
  private
@@ -7,47 +7,35 @@ module ActionPool
7
7
  def initialize
8
8
  super
9
9
  @wait = false
10
- @pause_lock = Mutex.new
11
- @empty_lock = Mutex.new
12
- @pause_guard = ConditionVariable.new
13
- @empty_guard = ConditionVariable.new
10
+ @pause_guard = Splib::Monitor.new
11
+ @empty_guard = Splib::Monitor.new
14
12
  end
15
13
  # Stop the queue from returning results to requesting
16
14
  # threads. Threads will wait for results until signalled
17
15
  def pause
18
- @pause_lock.synchronize{@wait = true}
16
+ @wait = true
19
17
  end
20
18
  # Allow the queue to return results. Any threads waiting
21
19
  # will have results given to them.
22
20
  def unpause
23
- @pause_lock.synchronize do
24
- @wait = false
25
- @pause_guard.broadcast
26
- end
21
+ @wait = false
22
+ @pause_guard.broadcast
27
23
  end
28
24
  # Check if queue needs to wait before returning
29
25
  def pop
30
- @pause_lock.synchronize do
31
- @pause_guard.wait(@pause_lock) if @wait
32
- end
26
+ @pause_guard.wait_while{ @wait }
33
27
  o = super
34
- @empty_lock.synchronize do
35
- @empty_guard.broadcast if empty?
36
- end
28
+ @empty_guard.broadcast if empty?
37
29
  return o
38
30
  end
39
31
  # Clear queue
40
32
  def clear
41
33
  super
42
- @empty_lock.synchronize do
43
- @empty_guard.broadcast
44
- end
34
+ @empty_guard.broadcast
45
35
  end
46
36
  # Park a thread here until queue is empty
47
37
  def wait_empty
48
- @empty_lock.synchronize do
49
- @empty_guard.wait(@empty_lock) if size > 0
50
- end
38
+ @empty_guard.wait_while{ size > 0 }
51
39
  end
52
40
  end
53
41
  end
@@ -13,6 +13,7 @@ module ActionPool
13
13
  # :a_timeout:: max time thread is allowed to work
14
14
  # :respond_thread:: thread to send execptions to
15
15
  # :logger:: LogHelper for logging messages
16
+ # :autostart:: Automatically start the thread
16
17
  # Create a new thread
17
18
  def initialize(args)
18
19
  raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
@@ -22,24 +23,45 @@ module ActionPool
22
23
  @respond_to = args[:respond_thread]
23
24
  @thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
24
25
  @action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
26
+ args[:autostart] = true unless args.has_key?(:autostart)
25
27
  @kill = false
26
- @logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
27
- @lock = Mutex.new
28
- @thread = ::Thread.new{ start_thread }
28
+ @logger = args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
29
+ @lock = Splib::Monitor.new
30
+ @action = nil
31
+ @thread = args[:autostart] ? ::Thread.new{ start_thread } : nil
29
32
  end
30
33
 
34
+ def start
35
+ @thread = ::Thread.new{ start_thread } if @thread.nil?
36
+ end
37
+
31
38
  # :force:: force the thread to stop
32
39
  # :wait:: wait for the thread to stop
33
40
  # Stop the thread
34
41
  def stop(*args)
35
42
  @kill = true
36
- @thread.raise Wakeup.new if args.include?(:force) || waiting?
43
+ if(args.include?(:force) || waiting?)
44
+ begin
45
+ @thread.raise Wakeup.new
46
+ rescue Wakeup
47
+ #ignore since we are the caller
48
+ end
49
+ sleep(0.01)
50
+ @thread.kill if @thread.alive?
51
+ end
37
52
  nil
38
53
  end
39
54
 
40
55
  # Currently waiting
41
56
  def waiting?
42
- @lock.synchronize{@status} == :wait
57
+ @action.nil?
58
+ # @status == :wait
59
+ end
60
+
61
+ # Currently running
62
+ def running?
63
+ !@action.nil?
64
+ # @status == :run
43
65
  end
44
66
 
45
67
  # Is the thread still alive
@@ -49,14 +71,21 @@ module ActionPool
49
71
 
50
72
  # Current thread status
51
73
  def status
52
- @lock.synchronize{ return @status }
74
+ @action
53
75
  end
54
-
55
- # arg:: :wait or :run
56
- # Set current status
57
- def status(arg)
58
- raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
59
- @lock.synchronize{ @status = arg }
76
+
77
+ # Join internal thread
78
+ def join
79
+ @thread.join(@action_timeout)
80
+ if(@thread.alive?)
81
+ @thread.kill
82
+ @thread.join
83
+ end
84
+ end
85
+
86
+ # Kill internal thread
87
+ def kill
88
+ @thread.kill
60
89
  end
61
90
 
62
91
  # Seconds thread will wait for input
@@ -96,20 +125,17 @@ module ActionPool
96
125
  begin
97
126
  @logger.info("New pool thread is starting (#{self})")
98
127
  until(@kill) do
99
- status(:wait)
100
128
  begin
101
- action = nil
129
+ @action = nil
102
130
  if(@pool.size > @pool.min && !@thread_timeout.zero?)
103
131
  Timeout::timeout(@thread_timeout) do
104
- action = @pool.action
132
+ @action = @pool.action
105
133
  end
106
134
  else
107
- action = @pool.action
135
+ @action = @pool.action
108
136
  end
109
- status(:run)
110
- run(action[0], action[1]) unless action.nil?
111
- status(:wait)
112
- rescue Timeout::Error => boom
137
+ run(@action[0], @action[1]) unless @action.nil?
138
+ rescue Timeout::Error
113
139
  @kill = true
114
140
  rescue Wakeup
115
141
  @logger.info("Thread #{::Thread.current} was woken up.")
@@ -131,7 +157,6 @@ module ActionPool
131
157
  ensure
132
158
  @logger.info("Pool thread is shutting down (#{self})")
133
159
  @pool.remove(self)
134
- @pool.create_thread
135
160
  end
136
161
  end
137
162
 
@@ -139,13 +164,14 @@ module ActionPool
139
164
  # args:: arguments to be passed to task
140
165
  # Run the task
141
166
  def run(action, args)
167
+ args = args.respond_to?(:fixed_flatten) ? args.fixed_flatten(1) : args.flatten(1)
142
168
  begin
143
169
  unless(@action_timeout.zero?)
144
170
  Timeout::timeout(@action_timeout) do
145
- action.call(*args[0])
171
+ action.call(*args)
146
172
  end
147
173
  else
148
- action.call(*args[0])
174
+ action.call(*args)
149
175
  end
150
176
  rescue Timeout::Error => boom
151
177
  @logger.warn("Pool thread reached max execution time for action: #{boom}")
@@ -28,6 +28,7 @@ class GeneralPoolTest < Test::Unit::TestCase
28
28
  jobs = [].fill(run,0,100)
29
29
  @pool.add_jobs(jobs)
30
30
  @pool.shutdown
31
+ sleep(0.01)
31
32
  assert_equal(100, a)
32
33
  @pool.shutdown(true)
33
34
  end
@@ -36,17 +37,24 @@ class GeneralPoolTest < Test::Unit::TestCase
36
37
  output = nil
37
38
  @pool << [lambda{|x| output = x}, [2]]
38
39
  assert(2, output)
40
+ output = nil
39
41
  @pool.add_jobs([[lambda{|x| output = x}, [3]]])
40
42
  assert(3, output)
43
+ output = nil
41
44
  @pool << [lambda{|x,y| output = x+y}, [1,2]]
42
45
  assert(3, output)
43
- output = []
44
- @pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
45
- assert(output.include?(2))
46
- assert(output.include?(3))
47
- @pool << [lambda{|x,y| output = [x,y]}, ['test', [1,2]]]
48
- assert_equal(output[0], 'test')
49
- assert(output[1].is_a?(Array))
46
+ output = nil
47
+ arr = []
48
+ @pool.add_jobs([[lambda{|x,y| arr << x + y}, [1,1]], [lambda{|x| arr << x}, [3]]])
49
+ ::Thread.pass
50
+ sleep(0.01)
51
+ assert(arr.include?(2))
52
+ assert(arr.include?(3))
53
+ arr.clear
54
+ @pool << [lambda{|x,y| arr = [x,y]}, ['test', [1,2]]]
55
+ sleep(0.01)
56
+ assert_equal('test', arr[0])
57
+ assert(arr[1].is_a?(Array))
50
58
  @pool.shutdown(true)
51
59
  end
52
60
  end
@@ -9,26 +9,39 @@ class TimeoutPoolTest < Test::Unit::TestCase
9
9
  @pool.shutdown(true)
10
10
  end
11
11
  def test_actiontimeout
12
- @pool.action_timeout = 0.01
12
+ @pool.action_timeout = 0.25
13
13
  assert_equal(10, @pool.size)
14
14
  stop = false
15
- @pool.add_jobs [].fill(lambda{loop{ 1+1 }}, 0, 20)
16
- ::Thread.pass
17
- assert(@pool.working > 10)
15
+ output = []
16
+ 20.times do
17
+ @pool.process do
18
+ until(stop) do
19
+ output << 1
20
+ sleep(0.1)
21
+ end
22
+ end
23
+ end
24
+ assert(@pool.working >= 10)
25
+ assert_equal(20, output.size)
26
+ sleep(0.11)
18
27
  stop = true
19
- sleep(0.5)
20
- assert(@pool.working == 0)
28
+ sleep(0.1)
29
+ assert_equal(0, @pool.working)
30
+ assert_equal(40, output.size)
21
31
  @pool.shutdown(true)
22
32
  end
23
33
  def test_threadtimeout
24
- @pool.thread_timeout = 0.01
34
+ @pool.thread_timeout = 0.05
25
35
  assert_equal(10, @pool.size)
26
- lock = Mutex.new
27
- guard = ConditionVariable.new
28
- @pool.add_jobs [].fill(lambda{ lock.synchronize{ guard.wait(lock) } }, 0, 20)
36
+ t = [].fill(lambda{
37
+ begin
38
+ sleep(0.1)
39
+ rescue
40
+ end }, 0, 20)
41
+ @pool.add_jobs(t)
29
42
  ::Thread.pass
30
- assert_equal(30, @pool.size)
31
- lock.synchronize{ guard.broadcast }
43
+ sleep(0.01)
44
+ assert(@pool.size >= 20)
32
45
  ::Thread.pass
33
46
  sleep(0.1)
34
47
  assert(10, @pool.size)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: actionpool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - spox
@@ -9,10 +9,19 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2010-01-05 00:00:00 -08:00
12
+ date: 2010-01-13 00:00:00 -08:00
13
13
  default_executable:
14
- dependencies: []
15
-
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: splib
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ~>
22
+ - !ruby/object:Gem::Version
23
+ version: "1.4"
24
+ version:
16
25
  description: The ActionPool is an easy to use thread pool for ruby.
17
26
  email: spox@modspox.com
18
27
  executables: []
@@ -23,30 +32,24 @@ extra_rdoc_files:
23
32
  - README.rdoc
24
33
  - CHANGELOG
25
34
  files:
26
- - lib/actionpool/Pool.rb~
27
- - lib/actionpool/LogHelper.rb
28
- - lib/actionpool/Pool.rb
29
- - lib/actionpool/Thread.rb
30
- - lib/actionpool/Queue.rb~
31
- - lib/actionpool/Queue.rb
32
- - lib/actionpool/Thread.rb~
33
- - lib/actionpool.rb
34
- - tests/cases/shutdown.rb
35
- - tests/cases/nogrow.rb
35
+ - Rakefile
36
+ - actionpool.gemspec
36
37
  - tests/cases/general.rb
37
- - tests/cases/thread.rb
38
- - tests/cases/grow.rb
39
- - tests/cases/general.rb~
40
- - tests/cases/timeouts.rb
41
38
  - tests/cases/resize.rb
39
+ - tests/cases/nogrow.rb
40
+ - tests/cases/timeouts.rb
41
+ - tests/cases/thread.rb
42
+ - tests/cases/shutdown.rb
42
43
  - tests/cases/queue.rb
44
+ - tests/cases/grow.rb
43
45
  - tests/run_tests.rb
44
- - Rakefile
46
+ - lib/actionpool/Pool.rb
47
+ - lib/actionpool/Thread.rb
48
+ - lib/actionpool/Queue.rb
49
+ - lib/actionpool.rb
45
50
  - CHANGELOG
46
51
  - LICENSE
47
52
  - README.rdoc
48
- - actionpool.gemspec
49
- - test.rb
50
53
  has_rdoc: true
51
54
  homepage: http://github.com/spox/actionpool
52
55
  licenses: []
@@ -1,25 +0,0 @@
1
- module ActionPool
2
- class LogHelper
3
-
4
- def initialize(logger=nil)
5
- require 'logger'
6
- @logger = logger
7
- end
8
-
9
- def info(m)
10
- @logger.info(m) unless @logger.nil?
11
- end
12
-
13
- def warn(m)
14
- @logger.warn(m) unless @logger.nil?
15
- end
16
-
17
- def fatal(m)
18
- @logger.fatal(m) unless @logger.nil?
19
- end
20
-
21
- def error(m)
22
- @logger.error(m) unless @logger.nil?
23
- end
24
- end
25
- end
@@ -1,260 +0,0 @@
1
- require 'actionpool/Thread'
2
- require 'actionpool/Queue'
3
- require 'actionpool/LogHelper'
4
- require 'thread'
5
-
6
- module ActionPool
7
- # Raised when pool is closed
8
- class PoolClosed < StandardError
9
- end
10
- class Pool
11
-
12
- # :min_threads:: minimum number of threads in pool
13
- # :max_threads:: maximum number of threads in pool
14
- # :t_to:: thread timeout waiting for action to process
15
- # :a_to:: maximum time action may be worked on before aborting
16
- # :logger:: logger to print logging messages to
17
- # Creates a new pool
18
- def initialize(args={})
19
- raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
20
- @logger = LogHelper.new(args[:logger])
21
- @queue = ActionPool::Queue.new
22
- @threads = []
23
- @lock = Mutex.new
24
- @thread_timeout = args[:t_to] ? args[:t_to] : 0
25
- @action_timeout = args[:a_to] ? args[:a_to] : 0
26
- @max_threads = args[:max_threads] ? args[:max_threads] : 100
27
- @min_threads = args[:min_threads] ? args[:min_threads] : 10
28
- @min_threads = @max_threads if @max_threads < @min_threads
29
- @respond_to = args[:respond_thread] || ::Thread.current
30
- @open = true
31
- create_thread
32
- end
33
-
34
- # Pool is closed
35
- def pool_closed?
36
- !@open
37
- end
38
-
39
- # Pool is open
40
- def pool_open?
41
- @open
42
- end
43
-
44
- # arg:: :open or :closed
45
- # Set pool status
46
- def status(arg)
47
- @open = arg == :open
48
- end
49
-
50
- # force:: force creation of a new thread
51
- # Create a new thread for pool. Returns newly created ActionPool::Thread or
52
- # nil if pool has reached maximum threads
53
- def create_thread(force=false)
54
- return if pool_closed?
55
- pt = nil
56
- @lock.synchronize do
57
- if(@threads.size < @max_threads || force)
58
- @logger.info('Pool is creating a new thread')
59
- (min - size > 0 ? min - size : 1).times do |i|
60
- pt = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
61
- @threads << pt
62
- end
63
- else
64
- @logger.info('Pool is at maximum size. Not creating new thread')
65
- end
66
- end
67
- return pt
68
- end
69
-
70
- # force:: force immediate stop
71
- # Stop the pool
72
- def shutdown(force=false)
73
- args = [:wait]
74
- args += [:force] if force
75
- @logger.info("Pool is now shutting down #{force ? 'using force' : ''}")
76
- @queue.wait_empty
77
- while(t = @threads.pop) do
78
- t.stop(*args)
79
- end
80
- nil
81
- end
82
-
83
- # action:: proc to be executed or array of [proc, [*args]]
84
- # Add a new proc/lambda to be executed (alias for queue)
85
- def <<(action)
86
- raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
87
- case action
88
- when Proc
89
- queue(action)
90
- when Array
91
- raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless action.size == 2 and action[0].is_a?(Proc) and action[1].is_a?(Array)
92
- queue(action[0], action[1])
93
- else
94
- raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
95
- end
96
- nil
97
- end
98
-
99
- # action:: proc to be executed
100
- # Add a new proc/lambda to be executed
101
- def queue(action, *args)
102
- raise ArgumentError.new('Expecting block') unless action.is_a?(Proc)
103
- @queue << [action, args]
104
- ::Thread.pass
105
- create_thread if @queue.num_waiting < 1 # only start a new thread if we need it
106
- end
107
-
108
- # jobs:: Array of proc/lambdas
109
- # Will queue a list of jobs into the pool
110
- def add_jobs(jobs)
111
- raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
112
- @queue.pause
113
- begin
114
- jobs.each do |job|
115
- case job
116
- when Proc
117
- @queue << [job, []]
118
- when Array
119
- raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
120
- @queue << [job.unshift, job]
121
- else
122
- raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
123
- end
124
- end
125
- ensure
126
- create_thread
127
- @queue.unpause
128
- end
129
- true
130
- end
131
-
132
- # block:: block to process
133
- # Adds a block to be processed
134
- def process(*args, &block)
135
- queue(block, *args)
136
- nil
137
- end
138
-
139
- # Current size of pool
140
- def size
141
- @threads.size
142
- end
143
-
144
- # Maximum allowed number of threads
145
- def max
146
- @max_threads
147
- end
148
-
149
- # Minimum allowed number of threads
150
- def min
151
- @min_threads
152
- end
153
-
154
- # m:: new max
155
- # Set maximum number of threads
156
- def max=(m)
157
- m = m.to_i
158
- raise ArgumentError.new('Maximum value must be greater than 0') unless m > 0
159
- @max_threads = m
160
- @min_threads = m if m < @min_threads
161
- resize if m < size
162
- m
163
- end
164
-
165
- # m:: new min
166
- # Set minimum number of threads
167
- def min=(m)
168
- m = m.to_i
169
- raise ArgumentError.new("Minimum value must be greater than 0 and less than or equal to maximum (#{max})") unless m > 0 && m <= max
170
- @min_threads = m
171
- m
172
- end
173
-
174
- # t:: ActionPool::Thread to remove
175
- # Removes a thread from the pool
176
- def remove(t)
177
- raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
178
- t.stop
179
- if(@threads.include?(t))
180
- @threads.delete(t)
181
- return true
182
- else
183
- return false
184
- end
185
- end
186
-
187
- # Maximum number of seconds a thread
188
- # is allowed to idle in the pool.
189
- # (nil means thread life is infinite)
190
- def thread_timeout
191
- @thread_timeout
192
- end
193
-
194
- # Maximum number of seconds a thread
195
- # is allowed to work on a given action
196
- # (nil means thread is given unlimited
197
- # time to work on action)
198
- def action_timeout
199
- @action_timeout
200
- end
201
-
202
- # t:: timeout in seconds (nil for infinite)
203
- # Set maximum allowed time thead may idle in pool
204
- def thread_timeout=(t)
205
- t = t.to_f
206
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
207
- @thread_timeout = t
208
- @threads.each{|thread|thread.thread_timeout = t}
209
- t
210
- end
211
-
212
- # t:: timeout in seconds (nil for infinte)
213
- # Set maximum allowed time thread may work
214
- # on a given action
215
- def action_timeout=(t)
216
- t = t.to_f
217
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
218
- @action_timeout = t
219
- @threads.each{|thread|thread.action_timeout = t}
220
- t
221
- end
222
-
223
- # Returns the next action to be processed
224
- def action
225
- @queue.pop
226
- end
227
-
228
- # Number of actions in the queue
229
- def action_size
230
- @queue.size
231
- end
232
-
233
- # Flush the thread pool. Mainly used for forcibly resizing
234
- # the pool if existing threads have a long thread life waiting
235
- # for input.
236
- def flush
237
- lock = Mutex.new
238
- guard = ConditionVariable.new
239
- @threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
240
- Thread.pass
241
- sleep(0.01)
242
- lock.synchronize{ guard.broadcast }
243
- end
244
-
245
- private
246
-
247
- # Resize the pool
248
- def resize
249
- @logger.info("Pool is being resized to stated maximum: #{max}")
250
- until(size <= max) do
251
- t = nil
252
- t = @threads.find{|t|t.waiting?}
253
- t = @threads.shift unless t
254
- t.stop
255
- end
256
- flush
257
- nil
258
- end
259
- end
260
- end
@@ -1,46 +0,0 @@
1
- require 'thread'
2
-
3
- module ActionPool
4
- # Adds a little bit extra functionality to the Queue class
5
- class Queue < ::Queue
6
- # Create a new Queue for the ActionPool::Pool
7
- def initialize
8
- super
9
- @wait = false
10
- @pause_lock = Mutex.new
11
- @emtpy_lock = Mutex.new
12
- @pause_guard = ConditionVariable.new
13
- @empty_guard = ConditionVariable.new
14
- end
15
- # Stop the queue from returning results to requesting
16
- # threads. Threads will wait for results until signalled
17
- def pause
18
- @pause_lock.synchronize{@wait = true}
19
- end
20
- # Allow the queue to return results. Any threads waiting
21
- # will have results given to them.
22
- def unpause
23
- @pause_lock.synchronize do
24
- @wait = false
25
- @pause_guard.broadcast
26
- end
27
- end
28
- # Check if queue needs to wait before returning
29
- def pop
30
- @pause_lock.synchronize do
31
- @pause_guard.wait(@pause_lock) if @wait
32
- end
33
- o = super
34
- @empty_lock.synchronize do
35
- @empty_guard.broadcast if empty?
36
- end
37
- return o
38
- end
39
- # Park a thread here until queue is empty
40
- def wait_empty
41
- @empty_lock.synchronize do
42
- @empty_guard.wait(@empty_lock) if size > 0
43
- end
44
- end
45
- end
46
- end
@@ -1,146 +0,0 @@
1
- require 'timeout'
2
-
3
- module ActionPool
4
- # Exception class used for waking up a thread
5
- class Wakeup < StandardError
6
- end
7
-
8
- class Thread
9
- # :pool:: pool thread is associated with
10
- # :t_timeout:: max time a thread is allowed to wait for action
11
- # :a_timeout:: max time thread is allowed to work
12
- # :respond_thread:: thread to send execptions to
13
- # :logger:: LogHelper for logging messages
14
- # Create a new thread
15
- def initialize(args)
16
- raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
17
- raise ArgumentError.new('ActionPool::Thread requires a pool') unless args[:pool]
18
- raise ArgumentError.new('ActionPool::Thread requries thread to respond') unless args[:respond_thread]
19
- @pool = args[:pool]
20
- @respond_to = args[:respond_thread]
21
- @thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
22
- @action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
23
- @kill = false
24
- @logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
25
- @lock = Mutex.new
26
- @thread = ::Thread.new{ start_thread }
27
- end
28
-
29
- # :force:: force the thread to stop
30
- # :wait:: wait for the thread to stop
31
- # Stop the thread
32
- def stop(*args)
33
- @kill = true
34
- @thread.raise Wakeup.new if args.include?(:force) || waiting?
35
- nil
36
- end
37
-
38
- # Currently waiting
39
- def waiting?
40
- @status == :wait
41
- end
42
-
43
- # Is the thread still alive
44
- def alive?
45
- @thread.alive?
46
- end
47
-
48
- # Current thread status
49
- def status
50
- @lock.synchronize{ return @status }
51
- end
52
-
53
- # arg:: :wait or :run
54
- # Set current status
55
- def status(arg)
56
- raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
57
- @lock.synchronize{ @status = arg }
58
- end
59
-
60
- # Seconds thread will wait for input
61
- def thread_timeout
62
- @thread_timeout
63
- end
64
-
65
- # Seconds thread will spend working on a given task
66
- def action_timeout
67
- @action_timeout
68
- end
69
-
70
- # t:: seconds to wait for input (floats allow for values 0 < t < 1)
71
- # Set the maximum amount of time to wait for a task
72
- def thread_timeout=(t)
73
- t = t.to_f
74
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
75
- @thread_timeout = t
76
- t
77
- end
78
-
79
- # t:: seconds to work on a task (floats allow for values 0 < t < 1)
80
- # Set the maximum amount of time to work on a given task
81
- def action_timeout=(t)
82
- t = t.to_f
83
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
84
- @action_timeout = t
85
- t
86
- end
87
-
88
- private
89
-
90
- # Start our thread
91
- def start_thread
92
- begin
93
- @logger.info("New pool thread is starting (#{self})")
94
- until(@kill) do
95
- status(:wait)
96
- begin
97
- action = nil
98
- if(@pool.size > @pool.min)
99
- Timeout::timeout(@thread_timeout) do
100
- action = @pool.action
101
- end
102
- else
103
- action = @pool.action
104
- end
105
- status(:run)
106
- run(action[0], action[1]) unless action.nil?
107
- status(:wait)
108
- rescue Timeout::Error => boom
109
- @kill = true
110
- rescue Wakeup
111
- @logger.info("Thread #{::Thread.current} was woken up.")
112
- rescue Exception => boom
113
- @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
114
- @respond_to.raise boom
115
- end
116
- end
117
- rescue Wakeup
118
- @logger.info("Thread #{::Thread.current} was woken up.")
119
- rescue Exception => boom
120
- @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
121
- @respond_to.raise boom
122
- ensure
123
- @logger.info("Pool thread is shutting down (#{self})")
124
- @pool.remove(self)
125
- @pool.create_threads
126
- end
127
- end
128
-
129
- # action:: task to be run
130
- # args:: arguments to be passed to task
131
- # Run the task
132
- def run(action, args)
133
- begin
134
- if(@action_timeout > 0)
135
- Timeout::timeout(@action_timeout) do
136
- action.call(*args.flatten)
137
- end
138
- else
139
- action.call(*args[0])
140
- end
141
- rescue Timeout::Error => boom
142
- @logger.warn("Pool thread reached max execution time for action: #{boom}")
143
- end
144
- end
145
- end
146
- end
data/test.rb DELETED
@@ -1,7 +0,0 @@
1
- require 'actionpool'
2
-
3
- pool = ActionPool::Pool.new
4
-
5
- pool << [lambda{|x,y| puts "#{x}, #{y}"}, [1,[:a, :b]]]
6
-
7
- sleep(1)
@@ -1,45 +0,0 @@
1
- require 'actionpool'
2
- require 'test/unit'
3
-
4
- class GeneralPoolTest < Test::Unit::TestCase
5
- def setup
6
- @pool = ActionPool::Pool.new
7
- end
8
- def test_numbers
9
- assert_equal(10, @pool.size)
10
- assert_equal(10, @pool.min)
11
- assert_equal(100, @pool.max)
12
- assert_equal(0, @pool.action_timeout)
13
- assert_equal(0, @pool.thread_timeout)
14
- assert_equal(0, @pool.action_size)
15
- end
16
- def test_output
17
- a = 0
18
- lock = Mutex.new
19
- run = lambda{ lock.synchronize{ a += 1 } }
20
- 100.times{ @pool << run }
21
- @pool.shutdown
22
- assert_equal(100, a)
23
- a = 0
24
- jobs = [].fill(run,0,100)
25
- @pool.add_jobs(jobs)
26
- @pool.shutdown
27
- assert_equal(100, a)
28
- @pool.shutdown(true)
29
- end
30
- def test_args
31
- output = nil
32
- @pool << [lambda{|x| output = x}, [2]]
33
- assert(2, output)
34
- @pool.add_jobs([[lambda{|x| output = x}, [3]]])
35
- assert(3, output)
36
- @pool << [lambda{|x,y| output = x+y}, [1,2]]
37
- # assert(3, output)
38
- sleep(0.01)
39
- output = []
40
- @pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
41
- # assert(output.include?(2))
42
- # assert(output.include?(3))
43
- @pool.shutdown(true)
44
- end
45
- end