actionpool 0.2.2 → 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
data/CHANGELOG CHANGED
@@ -1,3 +1,9 @@
1
+ 0.2.3
2
+ - Better thread destroying when removed normally and forcibly
3
+ - Flushing threads to ensure proper joining
4
+ - Fixed splatting for ruby < 1.9
5
+ - Removed the useless logger wrapper
6
+ - Use a simple monitor
1
7
  0.2.2
2
8
  - Better thread management within the pool
3
9
  - Restart thread timeouts when modified
@@ -4,7 +4,7 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
4
4
 
5
5
  === install (easy):
6
6
 
7
- gem install ActionPool
7
+ gem install actionpool
8
8
 
9
9
  === install (less easy):
10
10
 
@@ -17,6 +17,15 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
17
17
 
18
18
  {rip}[http://hellorip.com/about.html] makes it easy to install directly from a github repository.
19
19
 
20
+ === Testing
21
+
22
+ ActionPool is currently tested on:
23
+
24
+ * Ruby 1.8.6-p383
25
+ * Ruby 1.8.7-p248
26
+ * Ruby 1.9.1-p376
27
+ * JRuby 1.4.0
28
+
20
29
  == Documentation
21
30
 
22
31
  {rdocs}[http://allgems.ruby-forum.com/gems/ActionPool/]
@@ -2,7 +2,7 @@ spec = Gem::Specification.new do |s|
2
2
  s.name = 'actionpool'
3
3
  s.author = %q(spox)
4
4
  s.email = %q(spox@modspox.com)
5
- s.version = '0.2.2'
5
+ s.version = '0.2.3'
6
6
  s.summary = %q(Thread Pool)
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.files = Dir['**/*']
@@ -10,6 +10,7 @@ spec = Gem::Specification.new do |s|
10
10
  s.extra_rdoc_files = %w(README.rdoc CHANGELOG)
11
11
  s.require_paths = %w(lib)
12
12
  s.required_ruby_version = '>= 1.8.6'
13
+ s.add_dependency 'splib', '~> 1.4'
13
14
  s.homepage = %q(http://github.com/spox/actionpool)
14
15
  s.description = "The ActionPool is an easy to use thread pool for ruby."
15
16
  end
@@ -1,7 +1,10 @@
1
+ require 'rubygems'
1
2
  begin
2
3
  require 'fastthread'
3
4
  rescue LoadError
4
5
  # we don't care if it's available
5
6
  # just load it if it's around
6
7
  end
8
+ require 'splib'
9
+ Splib.load :Array, :Monitor
7
10
  require 'actionpool/Pool'
@@ -1,6 +1,6 @@
1
1
  require 'actionpool/Thread'
2
2
  require 'actionpool/Queue'
3
- require 'actionpool/LogHelper'
3
+ require 'logger'
4
4
  require 'thread'
5
5
 
6
6
  module ActionPool
@@ -17,10 +17,10 @@ module ActionPool
17
17
  # Creates a new pool
18
18
  def initialize(args={})
19
19
  raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
20
- @logger = LogHelper.new(args[:logger])
20
+ @logger = args[:logger] && args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
21
21
  @queue = ActionPool::Queue.new
22
22
  @threads = []
23
- @lock = Mutex.new
23
+ @lock = Splib::Monitor.new
24
24
  @thread_timeout = args[:t_to] ? args[:t_to] : 0
25
25
  @action_timeout = args[:a_to] ? args[:a_to] : 0
26
26
  @max_threads = args[:max_threads] ? args[:max_threads] : 100
@@ -48,36 +48,44 @@ module ActionPool
48
48
  fill_pool if @open
49
49
  end
50
50
 
51
- # args:: :force forces a new thread. :nowait will create a thread if threads are waiting
51
+ # args:: :force forces a new thread.
52
+ # :nowait will create a thread if threads are waiting
52
53
  # Create a new thread for pool.
53
- # Returns newly created thread of nil if pool is at maximum size
54
+ # Returns newly created thread or nil if pool is at maximum size
54
55
  def create_thread(*args)
55
56
  return if pool_closed?
56
57
  thread = nil
57
58
  @lock.synchronize do
58
59
  if(((size == working || args.include?(:nowait)) && @threads.size < @max_threads) || args.include?(:force))
59
- thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
60
+ thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
61
+ :t_timeout => @thread_timeout, :logger => @logger, :autostart => false)
60
62
  @threads << thread
61
63
  end
62
64
  end
63
- return thread
65
+ thread.start if thread
66
+ thread
64
67
  end
65
68
 
66
69
  # Fills the pool with the minimum number of threads
67
70
  # Returns array of created threads
68
71
  def fill_pool
69
72
  threads = []
70
- @lock.synchronize do
71
- required = min - size
72
- if(required > 0)
73
- required.times do
74
- thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
75
- @threads << thread
76
- threads << thread
73
+ if(@open)
74
+ @lock.synchronize do
75
+ required = min - size
76
+ if(required > 0)
77
+ required.times do
78
+ thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to,
79
+ :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger,
80
+ :autostart => false)
81
+ @threads << thread
82
+ threads << thread
83
+ end
77
84
  end
78
85
  end
79
86
  end
80
- return threads
87
+ threads.each{|t|t.start}
88
+ threads
81
89
  end
82
90
 
83
91
  # force:: force immediate stop
@@ -92,6 +100,10 @@ module ActionPool
92
100
  while(t = @threads.pop) do
93
101
  t.stop(*args)
94
102
  end
103
+ unless(force)
104
+ flush
105
+ @threads.each{|t|t.join}
106
+ end
95
107
  nil
96
108
  end
97
109
 
@@ -193,12 +205,10 @@ module ActionPool
193
205
  def remove(t)
194
206
  raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
195
207
  t.stop
196
- if(@threads.include?(t))
197
- @threads.delete(t)
198
- return true
199
- else
200
- return false
201
- end
208
+ del = @threads.include?(t)
209
+ @threads.delete(t) if del
210
+ fill_pool
211
+ del
202
212
  end
203
213
 
204
214
  # Maximum number of seconds a thread
@@ -251,17 +261,20 @@ module ActionPool
251
261
  # the pool if existing threads have a long thread life waiting
252
262
  # for input.
253
263
  def flush
254
- lock = Mutex.new
255
- guard = ConditionVariable.new
256
- @threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
257
- Thread.pass
264
+ mon = Splib::Monitor.new
265
+ @threads.size.times{ queue{ mon.wait } }
266
+ @queue.wait_empty
258
267
  sleep(0.01)
259
- lock.synchronize{ guard.broadcast }
268
+ mon.broadcast
260
269
  end
261
270
 
262
271
  # Returns current number of threads in the pool working
263
272
  def working
264
- @threads.find_all{|t|!t.waiting?}.size
273
+ @threads.select{|t|t.running?}.size
274
+ end
275
+
276
+ def thread_stats
277
+ @threads.map{|t|[t.object_id,t.status]}
265
278
  end
266
279
 
267
280
  private
@@ -7,47 +7,35 @@ module ActionPool
7
7
  def initialize
8
8
  super
9
9
  @wait = false
10
- @pause_lock = Mutex.new
11
- @empty_lock = Mutex.new
12
- @pause_guard = ConditionVariable.new
13
- @empty_guard = ConditionVariable.new
10
+ @pause_guard = Splib::Monitor.new
11
+ @empty_guard = Splib::Monitor.new
14
12
  end
15
13
  # Stop the queue from returning results to requesting
16
14
  # threads. Threads will wait for results until signalled
17
15
  def pause
18
- @pause_lock.synchronize{@wait = true}
16
+ @wait = true
19
17
  end
20
18
  # Allow the queue to return results. Any threads waiting
21
19
  # will have results given to them.
22
20
  def unpause
23
- @pause_lock.synchronize do
24
- @wait = false
25
- @pause_guard.broadcast
26
- end
21
+ @wait = false
22
+ @pause_guard.broadcast
27
23
  end
28
24
  # Check if queue needs to wait before returning
29
25
  def pop
30
- @pause_lock.synchronize do
31
- @pause_guard.wait(@pause_lock) if @wait
32
- end
26
+ @pause_guard.wait_while{ @wait }
33
27
  o = super
34
- @empty_lock.synchronize do
35
- @empty_guard.broadcast if empty?
36
- end
28
+ @empty_guard.broadcast if empty?
37
29
  return o
38
30
  end
39
31
  # Clear queue
40
32
  def clear
41
33
  super
42
- @empty_lock.synchronize do
43
- @empty_guard.broadcast
44
- end
34
+ @empty_guard.broadcast
45
35
  end
46
36
  # Park a thread here until queue is empty
47
37
  def wait_empty
48
- @empty_lock.synchronize do
49
- @empty_guard.wait(@empty_lock) if size > 0
50
- end
38
+ @empty_guard.wait_while{ size > 0 }
51
39
  end
52
40
  end
53
41
  end
@@ -13,6 +13,7 @@ module ActionPool
13
13
  # :a_timeout:: max time thread is allowed to work
14
14
  # :respond_thread:: thread to send execptions to
15
15
  # :logger:: LogHelper for logging messages
16
+ # :autostart:: Automatically start the thread
16
17
  # Create a new thread
17
18
  def initialize(args)
18
19
  raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
@@ -22,24 +23,45 @@ module ActionPool
22
23
  @respond_to = args[:respond_thread]
23
24
  @thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
24
25
  @action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
26
+ args[:autostart] = true unless args.has_key?(:autostart)
25
27
  @kill = false
26
- @logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
27
- @lock = Mutex.new
28
- @thread = ::Thread.new{ start_thread }
28
+ @logger = args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
29
+ @lock = Splib::Monitor.new
30
+ @action = nil
31
+ @thread = args[:autostart] ? ::Thread.new{ start_thread } : nil
29
32
  end
30
33
 
34
+ def start
35
+ @thread = ::Thread.new{ start_thread } if @thread.nil?
36
+ end
37
+
31
38
  # :force:: force the thread to stop
32
39
  # :wait:: wait for the thread to stop
33
40
  # Stop the thread
34
41
  def stop(*args)
35
42
  @kill = true
36
- @thread.raise Wakeup.new if args.include?(:force) || waiting?
43
+ if(args.include?(:force) || waiting?)
44
+ begin
45
+ @thread.raise Wakeup.new
46
+ rescue Wakeup
47
+ #ignore since we are the caller
48
+ end
49
+ sleep(0.01)
50
+ @thread.kill if @thread.alive?
51
+ end
37
52
  nil
38
53
  end
39
54
 
40
55
  # Currently waiting
41
56
  def waiting?
42
- @lock.synchronize{@status} == :wait
57
+ @action.nil?
58
+ # @status == :wait
59
+ end
60
+
61
+ # Currently running
62
+ def running?
63
+ !@action.nil?
64
+ # @status == :run
43
65
  end
44
66
 
45
67
  # Is the thread still alive
@@ -49,14 +71,21 @@ module ActionPool
49
71
 
50
72
  # Current thread status
51
73
  def status
52
- @lock.synchronize{ return @status }
74
+ @action
53
75
  end
54
-
55
- # arg:: :wait or :run
56
- # Set current status
57
- def status(arg)
58
- raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
59
- @lock.synchronize{ @status = arg }
76
+
77
+ # Join internal thread
78
+ def join
79
+ @thread.join(@action_timeout)
80
+ if(@thread.alive?)
81
+ @thread.kill
82
+ @thread.join
83
+ end
84
+ end
85
+
86
+ # Kill internal thread
87
+ def kill
88
+ @thread.kill
60
89
  end
61
90
 
62
91
  # Seconds thread will wait for input
@@ -96,20 +125,17 @@ module ActionPool
96
125
  begin
97
126
  @logger.info("New pool thread is starting (#{self})")
98
127
  until(@kill) do
99
- status(:wait)
100
128
  begin
101
- action = nil
129
+ @action = nil
102
130
  if(@pool.size > @pool.min && !@thread_timeout.zero?)
103
131
  Timeout::timeout(@thread_timeout) do
104
- action = @pool.action
132
+ @action = @pool.action
105
133
  end
106
134
  else
107
- action = @pool.action
135
+ @action = @pool.action
108
136
  end
109
- status(:run)
110
- run(action[0], action[1]) unless action.nil?
111
- status(:wait)
112
- rescue Timeout::Error => boom
137
+ run(@action[0], @action[1]) unless @action.nil?
138
+ rescue Timeout::Error
113
139
  @kill = true
114
140
  rescue Wakeup
115
141
  @logger.info("Thread #{::Thread.current} was woken up.")
@@ -131,7 +157,6 @@ module ActionPool
131
157
  ensure
132
158
  @logger.info("Pool thread is shutting down (#{self})")
133
159
  @pool.remove(self)
134
- @pool.create_thread
135
160
  end
136
161
  end
137
162
 
@@ -139,13 +164,14 @@ module ActionPool
139
164
  # args:: arguments to be passed to task
140
165
  # Run the task
141
166
  def run(action, args)
167
+ args = args.respond_to?(:fixed_flatten) ? args.fixed_flatten(1) : args.flatten(1)
142
168
  begin
143
169
  unless(@action_timeout.zero?)
144
170
  Timeout::timeout(@action_timeout) do
145
- action.call(*args[0])
171
+ action.call(*args)
146
172
  end
147
173
  else
148
- action.call(*args[0])
174
+ action.call(*args)
149
175
  end
150
176
  rescue Timeout::Error => boom
151
177
  @logger.warn("Pool thread reached max execution time for action: #{boom}")
@@ -28,6 +28,7 @@ class GeneralPoolTest < Test::Unit::TestCase
28
28
  jobs = [].fill(run,0,100)
29
29
  @pool.add_jobs(jobs)
30
30
  @pool.shutdown
31
+ sleep(0.01)
31
32
  assert_equal(100, a)
32
33
  @pool.shutdown(true)
33
34
  end
@@ -36,17 +37,24 @@ class GeneralPoolTest < Test::Unit::TestCase
36
37
  output = nil
37
38
  @pool << [lambda{|x| output = x}, [2]]
38
39
  assert(2, output)
40
+ output = nil
39
41
  @pool.add_jobs([[lambda{|x| output = x}, [3]]])
40
42
  assert(3, output)
43
+ output = nil
41
44
  @pool << [lambda{|x,y| output = x+y}, [1,2]]
42
45
  assert(3, output)
43
- output = []
44
- @pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
45
- assert(output.include?(2))
46
- assert(output.include?(3))
47
- @pool << [lambda{|x,y| output = [x,y]}, ['test', [1,2]]]
48
- assert_equal(output[0], 'test')
49
- assert(output[1].is_a?(Array))
46
+ output = nil
47
+ arr = []
48
+ @pool.add_jobs([[lambda{|x,y| arr << x + y}, [1,1]], [lambda{|x| arr << x}, [3]]])
49
+ ::Thread.pass
50
+ sleep(0.01)
51
+ assert(arr.include?(2))
52
+ assert(arr.include?(3))
53
+ arr.clear
54
+ @pool << [lambda{|x,y| arr = [x,y]}, ['test', [1,2]]]
55
+ sleep(0.01)
56
+ assert_equal('test', arr[0])
57
+ assert(arr[1].is_a?(Array))
50
58
  @pool.shutdown(true)
51
59
  end
52
60
  end
@@ -9,26 +9,39 @@ class TimeoutPoolTest < Test::Unit::TestCase
9
9
  @pool.shutdown(true)
10
10
  end
11
11
  def test_actiontimeout
12
- @pool.action_timeout = 0.01
12
+ @pool.action_timeout = 0.25
13
13
  assert_equal(10, @pool.size)
14
14
  stop = false
15
- @pool.add_jobs [].fill(lambda{loop{ 1+1 }}, 0, 20)
16
- ::Thread.pass
17
- assert(@pool.working > 10)
15
+ output = []
16
+ 20.times do
17
+ @pool.process do
18
+ until(stop) do
19
+ output << 1
20
+ sleep(0.1)
21
+ end
22
+ end
23
+ end
24
+ assert(@pool.working >= 10)
25
+ assert_equal(20, output.size)
26
+ sleep(0.11)
18
27
  stop = true
19
- sleep(0.5)
20
- assert(@pool.working == 0)
28
+ sleep(0.1)
29
+ assert_equal(0, @pool.working)
30
+ assert_equal(40, output.size)
21
31
  @pool.shutdown(true)
22
32
  end
23
33
  def test_threadtimeout
24
- @pool.thread_timeout = 0.01
34
+ @pool.thread_timeout = 0.05
25
35
  assert_equal(10, @pool.size)
26
- lock = Mutex.new
27
- guard = ConditionVariable.new
28
- @pool.add_jobs [].fill(lambda{ lock.synchronize{ guard.wait(lock) } }, 0, 20)
36
+ t = [].fill(lambda{
37
+ begin
38
+ sleep(0.1)
39
+ rescue
40
+ end }, 0, 20)
41
+ @pool.add_jobs(t)
29
42
  ::Thread.pass
30
- assert_equal(30, @pool.size)
31
- lock.synchronize{ guard.broadcast }
43
+ sleep(0.01)
44
+ assert(@pool.size >= 20)
32
45
  ::Thread.pass
33
46
  sleep(0.1)
34
47
  assert(10, @pool.size)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: actionpool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - spox
@@ -9,10 +9,19 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2010-01-05 00:00:00 -08:00
12
+ date: 2010-01-13 00:00:00 -08:00
13
13
  default_executable:
14
- dependencies: []
15
-
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: splib
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ~>
22
+ - !ruby/object:Gem::Version
23
+ version: "1.4"
24
+ version:
16
25
  description: The ActionPool is an easy to use thread pool for ruby.
17
26
  email: spox@modspox.com
18
27
  executables: []
@@ -23,30 +32,24 @@ extra_rdoc_files:
23
32
  - README.rdoc
24
33
  - CHANGELOG
25
34
  files:
26
- - lib/actionpool/Pool.rb~
27
- - lib/actionpool/LogHelper.rb
28
- - lib/actionpool/Pool.rb
29
- - lib/actionpool/Thread.rb
30
- - lib/actionpool/Queue.rb~
31
- - lib/actionpool/Queue.rb
32
- - lib/actionpool/Thread.rb~
33
- - lib/actionpool.rb
34
- - tests/cases/shutdown.rb
35
- - tests/cases/nogrow.rb
35
+ - Rakefile
36
+ - actionpool.gemspec
36
37
  - tests/cases/general.rb
37
- - tests/cases/thread.rb
38
- - tests/cases/grow.rb
39
- - tests/cases/general.rb~
40
- - tests/cases/timeouts.rb
41
38
  - tests/cases/resize.rb
39
+ - tests/cases/nogrow.rb
40
+ - tests/cases/timeouts.rb
41
+ - tests/cases/thread.rb
42
+ - tests/cases/shutdown.rb
42
43
  - tests/cases/queue.rb
44
+ - tests/cases/grow.rb
43
45
  - tests/run_tests.rb
44
- - Rakefile
46
+ - lib/actionpool/Pool.rb
47
+ - lib/actionpool/Thread.rb
48
+ - lib/actionpool/Queue.rb
49
+ - lib/actionpool.rb
45
50
  - CHANGELOG
46
51
  - LICENSE
47
52
  - README.rdoc
48
- - actionpool.gemspec
49
- - test.rb
50
53
  has_rdoc: true
51
54
  homepage: http://github.com/spox/actionpool
52
55
  licenses: []
@@ -1,25 +0,0 @@
1
- module ActionPool
2
- class LogHelper
3
-
4
- def initialize(logger=nil)
5
- require 'logger'
6
- @logger = logger
7
- end
8
-
9
- def info(m)
10
- @logger.info(m) unless @logger.nil?
11
- end
12
-
13
- def warn(m)
14
- @logger.warn(m) unless @logger.nil?
15
- end
16
-
17
- def fatal(m)
18
- @logger.fatal(m) unless @logger.nil?
19
- end
20
-
21
- def error(m)
22
- @logger.error(m) unless @logger.nil?
23
- end
24
- end
25
- end
@@ -1,260 +0,0 @@
1
- require 'actionpool/Thread'
2
- require 'actionpool/Queue'
3
- require 'actionpool/LogHelper'
4
- require 'thread'
5
-
6
- module ActionPool
7
- # Raised when pool is closed
8
- class PoolClosed < StandardError
9
- end
10
- class Pool
11
-
12
- # :min_threads:: minimum number of threads in pool
13
- # :max_threads:: maximum number of threads in pool
14
- # :t_to:: thread timeout waiting for action to process
15
- # :a_to:: maximum time action may be worked on before aborting
16
- # :logger:: logger to print logging messages to
17
- # Creates a new pool
18
- def initialize(args={})
19
- raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
20
- @logger = LogHelper.new(args[:logger])
21
- @queue = ActionPool::Queue.new
22
- @threads = []
23
- @lock = Mutex.new
24
- @thread_timeout = args[:t_to] ? args[:t_to] : 0
25
- @action_timeout = args[:a_to] ? args[:a_to] : 0
26
- @max_threads = args[:max_threads] ? args[:max_threads] : 100
27
- @min_threads = args[:min_threads] ? args[:min_threads] : 10
28
- @min_threads = @max_threads if @max_threads < @min_threads
29
- @respond_to = args[:respond_thread] || ::Thread.current
30
- @open = true
31
- create_thread
32
- end
33
-
34
- # Pool is closed
35
- def pool_closed?
36
- !@open
37
- end
38
-
39
- # Pool is open
40
- def pool_open?
41
- @open
42
- end
43
-
44
- # arg:: :open or :closed
45
- # Set pool status
46
- def status(arg)
47
- @open = arg == :open
48
- end
49
-
50
- # force:: force creation of a new thread
51
- # Create a new thread for pool. Returns newly created ActionPool::Thread or
52
- # nil if pool has reached maximum threads
53
- def create_thread(force=false)
54
- return if pool_closed?
55
- pt = nil
56
- @lock.synchronize do
57
- if(@threads.size < @max_threads || force)
58
- @logger.info('Pool is creating a new thread')
59
- (min - size > 0 ? min - size : 1).times do |i|
60
- pt = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
61
- @threads << pt
62
- end
63
- else
64
- @logger.info('Pool is at maximum size. Not creating new thread')
65
- end
66
- end
67
- return pt
68
- end
69
-
70
- # force:: force immediate stop
71
- # Stop the pool
72
- def shutdown(force=false)
73
- args = [:wait]
74
- args += [:force] if force
75
- @logger.info("Pool is now shutting down #{force ? 'using force' : ''}")
76
- @queue.wait_empty
77
- while(t = @threads.pop) do
78
- t.stop(*args)
79
- end
80
- nil
81
- end
82
-
83
- # action:: proc to be executed or array of [proc, [*args]]
84
- # Add a new proc/lambda to be executed (alias for queue)
85
- def <<(action)
86
- raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
87
- case action
88
- when Proc
89
- queue(action)
90
- when Array
91
- raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless action.size == 2 and action[0].is_a?(Proc) and action[1].is_a?(Array)
92
- queue(action[0], action[1])
93
- else
94
- raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
95
- end
96
- nil
97
- end
98
-
99
- # action:: proc to be executed
100
- # Add a new proc/lambda to be executed
101
- def queue(action, *args)
102
- raise ArgumentError.new('Expecting block') unless action.is_a?(Proc)
103
- @queue << [action, args]
104
- ::Thread.pass
105
- create_thread if @queue.num_waiting < 1 # only start a new thread if we need it
106
- end
107
-
108
- # jobs:: Array of proc/lambdas
109
- # Will queue a list of jobs into the pool
110
- def add_jobs(jobs)
111
- raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
112
- @queue.pause
113
- begin
114
- jobs.each do |job|
115
- case job
116
- when Proc
117
- @queue << [job, []]
118
- when Array
119
- raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
120
- @queue << [job.unshift, job]
121
- else
122
- raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
123
- end
124
- end
125
- ensure
126
- create_thread
127
- @queue.unpause
128
- end
129
- true
130
- end
131
-
132
- # block:: block to process
133
- # Adds a block to be processed
134
- def process(*args, &block)
135
- queue(block, *args)
136
- nil
137
- end
138
-
139
- # Current size of pool
140
- def size
141
- @threads.size
142
- end
143
-
144
- # Maximum allowed number of threads
145
- def max
146
- @max_threads
147
- end
148
-
149
- # Minimum allowed number of threads
150
- def min
151
- @min_threads
152
- end
153
-
154
- # m:: new max
155
- # Set maximum number of threads
156
- def max=(m)
157
- m = m.to_i
158
- raise ArgumentError.new('Maximum value must be greater than 0') unless m > 0
159
- @max_threads = m
160
- @min_threads = m if m < @min_threads
161
- resize if m < size
162
- m
163
- end
164
-
165
- # m:: new min
166
- # Set minimum number of threads
167
- def min=(m)
168
- m = m.to_i
169
- raise ArgumentError.new("Minimum value must be greater than 0 and less than or equal to maximum (#{max})") unless m > 0 && m <= max
170
- @min_threads = m
171
- m
172
- end
173
-
174
- # t:: ActionPool::Thread to remove
175
- # Removes a thread from the pool
176
- def remove(t)
177
- raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
178
- t.stop
179
- if(@threads.include?(t))
180
- @threads.delete(t)
181
- return true
182
- else
183
- return false
184
- end
185
- end
186
-
187
- # Maximum number of seconds a thread
188
- # is allowed to idle in the pool.
189
- # (nil means thread life is infinite)
190
- def thread_timeout
191
- @thread_timeout
192
- end
193
-
194
- # Maximum number of seconds a thread
195
- # is allowed to work on a given action
196
- # (nil means thread is given unlimited
197
- # time to work on action)
198
- def action_timeout
199
- @action_timeout
200
- end
201
-
202
- # t:: timeout in seconds (nil for infinite)
203
- # Set maximum allowed time thead may idle in pool
204
- def thread_timeout=(t)
205
- t = t.to_f
206
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
207
- @thread_timeout = t
208
- @threads.each{|thread|thread.thread_timeout = t}
209
- t
210
- end
211
-
212
- # t:: timeout in seconds (nil for infinte)
213
- # Set maximum allowed time thread may work
214
- # on a given action
215
- def action_timeout=(t)
216
- t = t.to_f
217
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
218
- @action_timeout = t
219
- @threads.each{|thread|thread.action_timeout = t}
220
- t
221
- end
222
-
223
- # Returns the next action to be processed
224
- def action
225
- @queue.pop
226
- end
227
-
228
- # Number of actions in the queue
229
- def action_size
230
- @queue.size
231
- end
232
-
233
- # Flush the thread pool. Mainly used for forcibly resizing
234
- # the pool if existing threads have a long thread life waiting
235
- # for input.
236
- def flush
237
- lock = Mutex.new
238
- guard = ConditionVariable.new
239
- @threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
240
- Thread.pass
241
- sleep(0.01)
242
- lock.synchronize{ guard.broadcast }
243
- end
244
-
245
- private
246
-
247
- # Resize the pool
248
- def resize
249
- @logger.info("Pool is being resized to stated maximum: #{max}")
250
- until(size <= max) do
251
- t = nil
252
- t = @threads.find{|t|t.waiting?}
253
- t = @threads.shift unless t
254
- t.stop
255
- end
256
- flush
257
- nil
258
- end
259
- end
260
- end
@@ -1,46 +0,0 @@
1
- require 'thread'
2
-
3
- module ActionPool
4
- # Adds a little bit extra functionality to the Queue class
5
- class Queue < ::Queue
6
- # Create a new Queue for the ActionPool::Pool
7
- def initialize
8
- super
9
- @wait = false
10
- @pause_lock = Mutex.new
11
- @emtpy_lock = Mutex.new
12
- @pause_guard = ConditionVariable.new
13
- @empty_guard = ConditionVariable.new
14
- end
15
- # Stop the queue from returning results to requesting
16
- # threads. Threads will wait for results until signalled
17
- def pause
18
- @pause_lock.synchronize{@wait = true}
19
- end
20
- # Allow the queue to return results. Any threads waiting
21
- # will have results given to them.
22
- def unpause
23
- @pause_lock.synchronize do
24
- @wait = false
25
- @pause_guard.broadcast
26
- end
27
- end
28
- # Check if queue needs to wait before returning
29
- def pop
30
- @pause_lock.synchronize do
31
- @pause_guard.wait(@pause_lock) if @wait
32
- end
33
- o = super
34
- @empty_lock.synchronize do
35
- @empty_guard.broadcast if empty?
36
- end
37
- return o
38
- end
39
- # Park a thread here until queue is empty
40
- def wait_empty
41
- @empty_lock.synchronize do
42
- @empty_guard.wait(@empty_lock) if size > 0
43
- end
44
- end
45
- end
46
- end
@@ -1,146 +0,0 @@
1
- require 'timeout'
2
-
3
- module ActionPool
4
- # Exception class used for waking up a thread
5
- class Wakeup < StandardError
6
- end
7
-
8
- class Thread
9
- # :pool:: pool thread is associated with
10
- # :t_timeout:: max time a thread is allowed to wait for action
11
- # :a_timeout:: max time thread is allowed to work
12
- # :respond_thread:: thread to send execptions to
13
- # :logger:: LogHelper for logging messages
14
- # Create a new thread
15
- def initialize(args)
16
- raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
17
- raise ArgumentError.new('ActionPool::Thread requires a pool') unless args[:pool]
18
- raise ArgumentError.new('ActionPool::Thread requries thread to respond') unless args[:respond_thread]
19
- @pool = args[:pool]
20
- @respond_to = args[:respond_thread]
21
- @thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
22
- @action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
23
- @kill = false
24
- @logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
25
- @lock = Mutex.new
26
- @thread = ::Thread.new{ start_thread }
27
- end
28
-
29
- # :force:: force the thread to stop
30
- # :wait:: wait for the thread to stop
31
- # Stop the thread
32
- def stop(*args)
33
- @kill = true
34
- @thread.raise Wakeup.new if args.include?(:force) || waiting?
35
- nil
36
- end
37
-
38
- # Currently waiting
39
- def waiting?
40
- @status == :wait
41
- end
42
-
43
- # Is the thread still alive
44
- def alive?
45
- @thread.alive?
46
- end
47
-
48
- # Current thread status
49
- def status
50
- @lock.synchronize{ return @status }
51
- end
52
-
53
- # arg:: :wait or :run
54
- # Set current status
55
- def status(arg)
56
- raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
57
- @lock.synchronize{ @status = arg }
58
- end
59
-
60
- # Seconds thread will wait for input
61
- def thread_timeout
62
- @thread_timeout
63
- end
64
-
65
- # Seconds thread will spend working on a given task
66
- def action_timeout
67
- @action_timeout
68
- end
69
-
70
- # t:: seconds to wait for input (floats allow for values 0 < t < 1)
71
- # Set the maximum amount of time to wait for a task
72
- def thread_timeout=(t)
73
- t = t.to_f
74
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
75
- @thread_timeout = t
76
- t
77
- end
78
-
79
- # t:: seconds to work on a task (floats allow for values 0 < t < 1)
80
- # Set the maximum amount of time to work on a given task
81
- def action_timeout=(t)
82
- t = t.to_f
83
- raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
84
- @action_timeout = t
85
- t
86
- end
87
-
88
- private
89
-
90
- # Start our thread
91
- def start_thread
92
- begin
93
- @logger.info("New pool thread is starting (#{self})")
94
- until(@kill) do
95
- status(:wait)
96
- begin
97
- action = nil
98
- if(@pool.size > @pool.min)
99
- Timeout::timeout(@thread_timeout) do
100
- action = @pool.action
101
- end
102
- else
103
- action = @pool.action
104
- end
105
- status(:run)
106
- run(action[0], action[1]) unless action.nil?
107
- status(:wait)
108
- rescue Timeout::Error => boom
109
- @kill = true
110
- rescue Wakeup
111
- @logger.info("Thread #{::Thread.current} was woken up.")
112
- rescue Exception => boom
113
- @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
114
- @respond_to.raise boom
115
- end
116
- end
117
- rescue Wakeup
118
- @logger.info("Thread #{::Thread.current} was woken up.")
119
- rescue Exception => boom
120
- @logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
121
- @respond_to.raise boom
122
- ensure
123
- @logger.info("Pool thread is shutting down (#{self})")
124
- @pool.remove(self)
125
- @pool.create_threads
126
- end
127
- end
128
-
129
- # action:: task to be run
130
- # args:: arguments to be passed to task
131
- # Run the task
132
- def run(action, args)
133
- begin
134
- if(@action_timeout > 0)
135
- Timeout::timeout(@action_timeout) do
136
- action.call(*args.flatten)
137
- end
138
- else
139
- action.call(*args[0])
140
- end
141
- rescue Timeout::Error => boom
142
- @logger.warn("Pool thread reached max execution time for action: #{boom}")
143
- end
144
- end
145
- end
146
- end
data/test.rb DELETED
@@ -1,7 +0,0 @@
1
- require 'actionpool'
2
-
3
- pool = ActionPool::Pool.new
4
-
5
- pool << [lambda{|x,y| puts "#{x}, #{y}"}, [1,[:a, :b]]]
6
-
7
- sleep(1)
@@ -1,45 +0,0 @@
1
- require 'actionpool'
2
- require 'test/unit'
3
-
4
- class GeneralPoolTest < Test::Unit::TestCase
5
- def setup
6
- @pool = ActionPool::Pool.new
7
- end
8
- def test_numbers
9
- assert_equal(10, @pool.size)
10
- assert_equal(10, @pool.min)
11
- assert_equal(100, @pool.max)
12
- assert_equal(0, @pool.action_timeout)
13
- assert_equal(0, @pool.thread_timeout)
14
- assert_equal(0, @pool.action_size)
15
- end
16
- def test_output
17
- a = 0
18
- lock = Mutex.new
19
- run = lambda{ lock.synchronize{ a += 1 } }
20
- 100.times{ @pool << run }
21
- @pool.shutdown
22
- assert_equal(100, a)
23
- a = 0
24
- jobs = [].fill(run,0,100)
25
- @pool.add_jobs(jobs)
26
- @pool.shutdown
27
- assert_equal(100, a)
28
- @pool.shutdown(true)
29
- end
30
- def test_args
31
- output = nil
32
- @pool << [lambda{|x| output = x}, [2]]
33
- assert(2, output)
34
- @pool.add_jobs([[lambda{|x| output = x}, [3]]])
35
- assert(3, output)
36
- @pool << [lambda{|x,y| output = x+y}, [1,2]]
37
- # assert(3, output)
38
- sleep(0.01)
39
- output = []
40
- @pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
41
- # assert(output.include?(2))
42
- # assert(output.include?(3))
43
- @pool.shutdown(true)
44
- end
45
- end