actionpool 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +6 -0
- data/README.rdoc +10 -1
- data/actionpool.gemspec +2 -1
- data/lib/actionpool.rb +3 -0
- data/lib/actionpool/Pool.rb +40 -27
- data/lib/actionpool/Queue.rb +9 -21
- data/lib/actionpool/Thread.rb +49 -23
- data/tests/cases/general.rb +15 -7
- data/tests/cases/timeouts.rb +25 -12
- metadata +24 -21
- data/lib/actionpool/LogHelper.rb +0 -25
- data/lib/actionpool/Pool.rb~ +0 -260
- data/lib/actionpool/Queue.rb~ +0 -46
- data/lib/actionpool/Thread.rb~ +0 -146
- data/test.rb +0 -7
- data/tests/cases/general.rb~ +0 -45
data/CHANGELOG
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
0.2.3
|
2
|
+
- Better thread destroying when removed normally and forcibly
|
3
|
+
- Flushing threads to ensure proper joining
|
4
|
+
- Fixed splatting for ruby < 1.9
|
5
|
+
- Removed the useless logger wrapper
|
6
|
+
- Use a simple monitor
|
1
7
|
0.2.2
|
2
8
|
- Better thread management within the pool
|
3
9
|
- Restart thread timeouts when modified
|
data/README.rdoc
CHANGED
@@ -4,7 +4,7 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
|
|
4
4
|
|
5
5
|
=== install (easy):
|
6
6
|
|
7
|
-
gem install
|
7
|
+
gem install actionpool
|
8
8
|
|
9
9
|
=== install (less easy):
|
10
10
|
|
@@ -17,6 +17,15 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
|
|
17
17
|
|
18
18
|
{rip}[http://hellorip.com/about.html] makes it easy to install directly from a github repository.
|
19
19
|
|
20
|
+
=== Testing
|
21
|
+
|
22
|
+
ActionPool is currently tested on:
|
23
|
+
|
24
|
+
* Ruby 1.8.6-p383
|
25
|
+
* Ruby 1.8.7-p248
|
26
|
+
* Ruby 1.9.1-p376
|
27
|
+
* JRuby 1.4.0
|
28
|
+
|
20
29
|
== Documentation
|
21
30
|
|
22
31
|
{rdocs}[http://allgems.ruby-forum.com/gems/ActionPool/]
|
data/actionpool.gemspec
CHANGED
@@ -2,7 +2,7 @@ spec = Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'actionpool'
|
3
3
|
s.author = %q(spox)
|
4
4
|
s.email = %q(spox@modspox.com)
|
5
|
-
s.version = '0.2.
|
5
|
+
s.version = '0.2.3'
|
6
6
|
s.summary = %q(Thread Pool)
|
7
7
|
s.platform = Gem::Platform::RUBY
|
8
8
|
s.files = Dir['**/*']
|
@@ -10,6 +10,7 @@ spec = Gem::Specification.new do |s|
|
|
10
10
|
s.extra_rdoc_files = %w(README.rdoc CHANGELOG)
|
11
11
|
s.require_paths = %w(lib)
|
12
12
|
s.required_ruby_version = '>= 1.8.6'
|
13
|
+
s.add_dependency 'splib', '~> 1.4'
|
13
14
|
s.homepage = %q(http://github.com/spox/actionpool)
|
14
15
|
s.description = "The ActionPool is an easy to use thread pool for ruby."
|
15
16
|
end
|
data/lib/actionpool.rb
CHANGED
data/lib/actionpool/Pool.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'actionpool/Thread'
|
2
2
|
require 'actionpool/Queue'
|
3
|
-
require '
|
3
|
+
require 'logger'
|
4
4
|
require 'thread'
|
5
5
|
|
6
6
|
module ActionPool
|
@@ -17,10 +17,10 @@ module ActionPool
|
|
17
17
|
# Creates a new pool
|
18
18
|
def initialize(args={})
|
19
19
|
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
20
|
-
@logger =
|
20
|
+
@logger = args[:logger] && args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
|
21
21
|
@queue = ActionPool::Queue.new
|
22
22
|
@threads = []
|
23
|
-
@lock =
|
23
|
+
@lock = Splib::Monitor.new
|
24
24
|
@thread_timeout = args[:t_to] ? args[:t_to] : 0
|
25
25
|
@action_timeout = args[:a_to] ? args[:a_to] : 0
|
26
26
|
@max_threads = args[:max_threads] ? args[:max_threads] : 100
|
@@ -48,36 +48,44 @@ module ActionPool
|
|
48
48
|
fill_pool if @open
|
49
49
|
end
|
50
50
|
|
51
|
-
# args::
|
51
|
+
# args:: :force forces a new thread.
|
52
|
+
# :nowait will create a thread if threads are waiting
|
52
53
|
# Create a new thread for pool.
|
53
|
-
# Returns newly created thread
|
54
|
+
# Returns newly created thread or nil if pool is at maximum size
|
54
55
|
def create_thread(*args)
|
55
56
|
return if pool_closed?
|
56
57
|
thread = nil
|
57
58
|
@lock.synchronize do
|
58
59
|
if(((size == working || args.include?(:nowait)) && @threads.size < @max_threads) || args.include?(:force))
|
59
|
-
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
|
60
|
+
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
|
61
|
+
:t_timeout => @thread_timeout, :logger => @logger, :autostart => false)
|
60
62
|
@threads << thread
|
61
63
|
end
|
62
64
|
end
|
63
|
-
|
65
|
+
thread.start if thread
|
66
|
+
thread
|
64
67
|
end
|
65
68
|
|
66
69
|
# Fills the pool with the minimum number of threads
|
67
70
|
# Returns array of created threads
|
68
71
|
def fill_pool
|
69
72
|
threads = []
|
70
|
-
@
|
71
|
-
|
72
|
-
|
73
|
-
required
|
74
|
-
|
75
|
-
|
76
|
-
|
73
|
+
if(@open)
|
74
|
+
@lock.synchronize do
|
75
|
+
required = min - size
|
76
|
+
if(required > 0)
|
77
|
+
required.times do
|
78
|
+
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to,
|
79
|
+
:a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger,
|
80
|
+
:autostart => false)
|
81
|
+
@threads << thread
|
82
|
+
threads << thread
|
83
|
+
end
|
77
84
|
end
|
78
85
|
end
|
79
86
|
end
|
80
|
-
|
87
|
+
threads.each{|t|t.start}
|
88
|
+
threads
|
81
89
|
end
|
82
90
|
|
83
91
|
# force:: force immediate stop
|
@@ -92,6 +100,10 @@ module ActionPool
|
|
92
100
|
while(t = @threads.pop) do
|
93
101
|
t.stop(*args)
|
94
102
|
end
|
103
|
+
unless(force)
|
104
|
+
flush
|
105
|
+
@threads.each{|t|t.join}
|
106
|
+
end
|
95
107
|
nil
|
96
108
|
end
|
97
109
|
|
@@ -193,12 +205,10 @@ module ActionPool
|
|
193
205
|
def remove(t)
|
194
206
|
raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
|
195
207
|
t.stop
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
return false
|
201
|
-
end
|
208
|
+
del = @threads.include?(t)
|
209
|
+
@threads.delete(t) if del
|
210
|
+
fill_pool
|
211
|
+
del
|
202
212
|
end
|
203
213
|
|
204
214
|
# Maximum number of seconds a thread
|
@@ -251,17 +261,20 @@ module ActionPool
|
|
251
261
|
# the pool if existing threads have a long thread life waiting
|
252
262
|
# for input.
|
253
263
|
def flush
|
254
|
-
|
255
|
-
|
256
|
-
@
|
257
|
-
Thread.pass
|
264
|
+
mon = Splib::Monitor.new
|
265
|
+
@threads.size.times{ queue{ mon.wait } }
|
266
|
+
@queue.wait_empty
|
258
267
|
sleep(0.01)
|
259
|
-
|
268
|
+
mon.broadcast
|
260
269
|
end
|
261
270
|
|
262
271
|
# Returns current number of threads in the pool working
|
263
272
|
def working
|
264
|
-
@threads.
|
273
|
+
@threads.select{|t|t.running?}.size
|
274
|
+
end
|
275
|
+
|
276
|
+
def thread_stats
|
277
|
+
@threads.map{|t|[t.object_id,t.status]}
|
265
278
|
end
|
266
279
|
|
267
280
|
private
|
data/lib/actionpool/Queue.rb
CHANGED
@@ -7,47 +7,35 @@ module ActionPool
|
|
7
7
|
def initialize
|
8
8
|
super
|
9
9
|
@wait = false
|
10
|
-
@
|
11
|
-
@
|
12
|
-
@pause_guard = ConditionVariable.new
|
13
|
-
@empty_guard = ConditionVariable.new
|
10
|
+
@pause_guard = Splib::Monitor.new
|
11
|
+
@empty_guard = Splib::Monitor.new
|
14
12
|
end
|
15
13
|
# Stop the queue from returning results to requesting
|
16
14
|
# threads. Threads will wait for results until signalled
|
17
15
|
def pause
|
18
|
-
@
|
16
|
+
@wait = true
|
19
17
|
end
|
20
18
|
# Allow the queue to return results. Any threads waiting
|
21
19
|
# will have results given to them.
|
22
20
|
def unpause
|
23
|
-
@
|
24
|
-
|
25
|
-
@pause_guard.broadcast
|
26
|
-
end
|
21
|
+
@wait = false
|
22
|
+
@pause_guard.broadcast
|
27
23
|
end
|
28
24
|
# Check if queue needs to wait before returning
|
29
25
|
def pop
|
30
|
-
@
|
31
|
-
@pause_guard.wait(@pause_lock) if @wait
|
32
|
-
end
|
26
|
+
@pause_guard.wait_while{ @wait }
|
33
27
|
o = super
|
34
|
-
@
|
35
|
-
@empty_guard.broadcast if empty?
|
36
|
-
end
|
28
|
+
@empty_guard.broadcast if empty?
|
37
29
|
return o
|
38
30
|
end
|
39
31
|
# Clear queue
|
40
32
|
def clear
|
41
33
|
super
|
42
|
-
@
|
43
|
-
@empty_guard.broadcast
|
44
|
-
end
|
34
|
+
@empty_guard.broadcast
|
45
35
|
end
|
46
36
|
# Park a thread here until queue is empty
|
47
37
|
def wait_empty
|
48
|
-
@
|
49
|
-
@empty_guard.wait(@empty_lock) if size > 0
|
50
|
-
end
|
38
|
+
@empty_guard.wait_while{ size > 0 }
|
51
39
|
end
|
52
40
|
end
|
53
41
|
end
|
data/lib/actionpool/Thread.rb
CHANGED
@@ -13,6 +13,7 @@ module ActionPool
|
|
13
13
|
# :a_timeout:: max time thread is allowed to work
|
14
14
|
# :respond_thread:: thread to send execptions to
|
15
15
|
# :logger:: LogHelper for logging messages
|
16
|
+
# :autostart:: Automatically start the thread
|
16
17
|
# Create a new thread
|
17
18
|
def initialize(args)
|
18
19
|
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
@@ -22,24 +23,45 @@ module ActionPool
|
|
22
23
|
@respond_to = args[:respond_thread]
|
23
24
|
@thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
|
24
25
|
@action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
|
26
|
+
args[:autostart] = true unless args.has_key?(:autostart)
|
25
27
|
@kill = false
|
26
|
-
@logger = args[:logger].is_a?(
|
27
|
-
@lock =
|
28
|
-
@
|
28
|
+
@logger = args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
|
29
|
+
@lock = Splib::Monitor.new
|
30
|
+
@action = nil
|
31
|
+
@thread = args[:autostart] ? ::Thread.new{ start_thread } : nil
|
29
32
|
end
|
30
33
|
|
34
|
+
def start
|
35
|
+
@thread = ::Thread.new{ start_thread } if @thread.nil?
|
36
|
+
end
|
37
|
+
|
31
38
|
# :force:: force the thread to stop
|
32
39
|
# :wait:: wait for the thread to stop
|
33
40
|
# Stop the thread
|
34
41
|
def stop(*args)
|
35
42
|
@kill = true
|
36
|
-
|
43
|
+
if(args.include?(:force) || waiting?)
|
44
|
+
begin
|
45
|
+
@thread.raise Wakeup.new
|
46
|
+
rescue Wakeup
|
47
|
+
#ignore since we are the caller
|
48
|
+
end
|
49
|
+
sleep(0.01)
|
50
|
+
@thread.kill if @thread.alive?
|
51
|
+
end
|
37
52
|
nil
|
38
53
|
end
|
39
54
|
|
40
55
|
# Currently waiting
|
41
56
|
def waiting?
|
42
|
-
@
|
57
|
+
@action.nil?
|
58
|
+
# @status == :wait
|
59
|
+
end
|
60
|
+
|
61
|
+
# Currently running
|
62
|
+
def running?
|
63
|
+
!@action.nil?
|
64
|
+
# @status == :run
|
43
65
|
end
|
44
66
|
|
45
67
|
# Is the thread still alive
|
@@ -49,14 +71,21 @@ module ActionPool
|
|
49
71
|
|
50
72
|
# Current thread status
|
51
73
|
def status
|
52
|
-
@
|
74
|
+
@action
|
53
75
|
end
|
54
|
-
|
55
|
-
#
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
76
|
+
|
77
|
+
# Join internal thread
|
78
|
+
def join
|
79
|
+
@thread.join(@action_timeout)
|
80
|
+
if(@thread.alive?)
|
81
|
+
@thread.kill
|
82
|
+
@thread.join
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Kill internal thread
|
87
|
+
def kill
|
88
|
+
@thread.kill
|
60
89
|
end
|
61
90
|
|
62
91
|
# Seconds thread will wait for input
|
@@ -96,20 +125,17 @@ module ActionPool
|
|
96
125
|
begin
|
97
126
|
@logger.info("New pool thread is starting (#{self})")
|
98
127
|
until(@kill) do
|
99
|
-
status(:wait)
|
100
128
|
begin
|
101
|
-
action = nil
|
129
|
+
@action = nil
|
102
130
|
if(@pool.size > @pool.min && !@thread_timeout.zero?)
|
103
131
|
Timeout::timeout(@thread_timeout) do
|
104
|
-
action = @pool.action
|
132
|
+
@action = @pool.action
|
105
133
|
end
|
106
134
|
else
|
107
|
-
action = @pool.action
|
135
|
+
@action = @pool.action
|
108
136
|
end
|
109
|
-
|
110
|
-
|
111
|
-
status(:wait)
|
112
|
-
rescue Timeout::Error => boom
|
137
|
+
run(@action[0], @action[1]) unless @action.nil?
|
138
|
+
rescue Timeout::Error
|
113
139
|
@kill = true
|
114
140
|
rescue Wakeup
|
115
141
|
@logger.info("Thread #{::Thread.current} was woken up.")
|
@@ -131,7 +157,6 @@ module ActionPool
|
|
131
157
|
ensure
|
132
158
|
@logger.info("Pool thread is shutting down (#{self})")
|
133
159
|
@pool.remove(self)
|
134
|
-
@pool.create_thread
|
135
160
|
end
|
136
161
|
end
|
137
162
|
|
@@ -139,13 +164,14 @@ module ActionPool
|
|
139
164
|
# args:: arguments to be passed to task
|
140
165
|
# Run the task
|
141
166
|
def run(action, args)
|
167
|
+
args = args.respond_to?(:fixed_flatten) ? args.fixed_flatten(1) : args.flatten(1)
|
142
168
|
begin
|
143
169
|
unless(@action_timeout.zero?)
|
144
170
|
Timeout::timeout(@action_timeout) do
|
145
|
-
action.call(*args
|
171
|
+
action.call(*args)
|
146
172
|
end
|
147
173
|
else
|
148
|
-
action.call(*args
|
174
|
+
action.call(*args)
|
149
175
|
end
|
150
176
|
rescue Timeout::Error => boom
|
151
177
|
@logger.warn("Pool thread reached max execution time for action: #{boom}")
|
data/tests/cases/general.rb
CHANGED
@@ -28,6 +28,7 @@ class GeneralPoolTest < Test::Unit::TestCase
|
|
28
28
|
jobs = [].fill(run,0,100)
|
29
29
|
@pool.add_jobs(jobs)
|
30
30
|
@pool.shutdown
|
31
|
+
sleep(0.01)
|
31
32
|
assert_equal(100, a)
|
32
33
|
@pool.shutdown(true)
|
33
34
|
end
|
@@ -36,17 +37,24 @@ class GeneralPoolTest < Test::Unit::TestCase
|
|
36
37
|
output = nil
|
37
38
|
@pool << [lambda{|x| output = x}, [2]]
|
38
39
|
assert(2, output)
|
40
|
+
output = nil
|
39
41
|
@pool.add_jobs([[lambda{|x| output = x}, [3]]])
|
40
42
|
assert(3, output)
|
43
|
+
output = nil
|
41
44
|
@pool << [lambda{|x,y| output = x+y}, [1,2]]
|
42
45
|
assert(3, output)
|
43
|
-
output =
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
assert(
|
46
|
+
output = nil
|
47
|
+
arr = []
|
48
|
+
@pool.add_jobs([[lambda{|x,y| arr << x + y}, [1,1]], [lambda{|x| arr << x}, [3]]])
|
49
|
+
::Thread.pass
|
50
|
+
sleep(0.01)
|
51
|
+
assert(arr.include?(2))
|
52
|
+
assert(arr.include?(3))
|
53
|
+
arr.clear
|
54
|
+
@pool << [lambda{|x,y| arr = [x,y]}, ['test', [1,2]]]
|
55
|
+
sleep(0.01)
|
56
|
+
assert_equal('test', arr[0])
|
57
|
+
assert(arr[1].is_a?(Array))
|
50
58
|
@pool.shutdown(true)
|
51
59
|
end
|
52
60
|
end
|
data/tests/cases/timeouts.rb
CHANGED
@@ -9,26 +9,39 @@ class TimeoutPoolTest < Test::Unit::TestCase
|
|
9
9
|
@pool.shutdown(true)
|
10
10
|
end
|
11
11
|
def test_actiontimeout
|
12
|
-
@pool.action_timeout = 0.
|
12
|
+
@pool.action_timeout = 0.25
|
13
13
|
assert_equal(10, @pool.size)
|
14
14
|
stop = false
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
output = []
|
16
|
+
20.times do
|
17
|
+
@pool.process do
|
18
|
+
until(stop) do
|
19
|
+
output << 1
|
20
|
+
sleep(0.1)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
assert(@pool.working >= 10)
|
25
|
+
assert_equal(20, output.size)
|
26
|
+
sleep(0.11)
|
18
27
|
stop = true
|
19
|
-
sleep(0.
|
20
|
-
|
28
|
+
sleep(0.1)
|
29
|
+
assert_equal(0, @pool.working)
|
30
|
+
assert_equal(40, output.size)
|
21
31
|
@pool.shutdown(true)
|
22
32
|
end
|
23
33
|
def test_threadtimeout
|
24
|
-
@pool.thread_timeout = 0.
|
34
|
+
@pool.thread_timeout = 0.05
|
25
35
|
assert_equal(10, @pool.size)
|
26
|
-
|
27
|
-
|
28
|
-
|
36
|
+
t = [].fill(lambda{
|
37
|
+
begin
|
38
|
+
sleep(0.1)
|
39
|
+
rescue
|
40
|
+
end }, 0, 20)
|
41
|
+
@pool.add_jobs(t)
|
29
42
|
::Thread.pass
|
30
|
-
|
31
|
-
|
43
|
+
sleep(0.01)
|
44
|
+
assert(@pool.size >= 20)
|
32
45
|
::Thread.pass
|
33
46
|
sleep(0.1)
|
34
47
|
assert(10, @pool.size)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: actionpool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- spox
|
@@ -9,10 +9,19 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-01-
|
12
|
+
date: 2010-01-13 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
|
-
dependencies:
|
15
|
-
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: splib
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ~>
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "1.4"
|
24
|
+
version:
|
16
25
|
description: The ActionPool is an easy to use thread pool for ruby.
|
17
26
|
email: spox@modspox.com
|
18
27
|
executables: []
|
@@ -23,30 +32,24 @@ extra_rdoc_files:
|
|
23
32
|
- README.rdoc
|
24
33
|
- CHANGELOG
|
25
34
|
files:
|
26
|
-
-
|
27
|
-
-
|
28
|
-
- lib/actionpool/Pool.rb
|
29
|
-
- lib/actionpool/Thread.rb
|
30
|
-
- lib/actionpool/Queue.rb~
|
31
|
-
- lib/actionpool/Queue.rb
|
32
|
-
- lib/actionpool/Thread.rb~
|
33
|
-
- lib/actionpool.rb
|
34
|
-
- tests/cases/shutdown.rb
|
35
|
-
- tests/cases/nogrow.rb
|
35
|
+
- Rakefile
|
36
|
+
- actionpool.gemspec
|
36
37
|
- tests/cases/general.rb
|
37
|
-
- tests/cases/thread.rb
|
38
|
-
- tests/cases/grow.rb
|
39
|
-
- tests/cases/general.rb~
|
40
|
-
- tests/cases/timeouts.rb
|
41
38
|
- tests/cases/resize.rb
|
39
|
+
- tests/cases/nogrow.rb
|
40
|
+
- tests/cases/timeouts.rb
|
41
|
+
- tests/cases/thread.rb
|
42
|
+
- tests/cases/shutdown.rb
|
42
43
|
- tests/cases/queue.rb
|
44
|
+
- tests/cases/grow.rb
|
43
45
|
- tests/run_tests.rb
|
44
|
-
-
|
46
|
+
- lib/actionpool/Pool.rb
|
47
|
+
- lib/actionpool/Thread.rb
|
48
|
+
- lib/actionpool/Queue.rb
|
49
|
+
- lib/actionpool.rb
|
45
50
|
- CHANGELOG
|
46
51
|
- LICENSE
|
47
52
|
- README.rdoc
|
48
|
-
- actionpool.gemspec
|
49
|
-
- test.rb
|
50
53
|
has_rdoc: true
|
51
54
|
homepage: http://github.com/spox/actionpool
|
52
55
|
licenses: []
|
data/lib/actionpool/LogHelper.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
module ActionPool
|
2
|
-
class LogHelper
|
3
|
-
|
4
|
-
def initialize(logger=nil)
|
5
|
-
require 'logger'
|
6
|
-
@logger = logger
|
7
|
-
end
|
8
|
-
|
9
|
-
def info(m)
|
10
|
-
@logger.info(m) unless @logger.nil?
|
11
|
-
end
|
12
|
-
|
13
|
-
def warn(m)
|
14
|
-
@logger.warn(m) unless @logger.nil?
|
15
|
-
end
|
16
|
-
|
17
|
-
def fatal(m)
|
18
|
-
@logger.fatal(m) unless @logger.nil?
|
19
|
-
end
|
20
|
-
|
21
|
-
def error(m)
|
22
|
-
@logger.error(m) unless @logger.nil?
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
data/lib/actionpool/Pool.rb~
DELETED
@@ -1,260 +0,0 @@
|
|
1
|
-
require 'actionpool/Thread'
|
2
|
-
require 'actionpool/Queue'
|
3
|
-
require 'actionpool/LogHelper'
|
4
|
-
require 'thread'
|
5
|
-
|
6
|
-
module ActionPool
|
7
|
-
# Raised when pool is closed
|
8
|
-
class PoolClosed < StandardError
|
9
|
-
end
|
10
|
-
class Pool
|
11
|
-
|
12
|
-
# :min_threads:: minimum number of threads in pool
|
13
|
-
# :max_threads:: maximum number of threads in pool
|
14
|
-
# :t_to:: thread timeout waiting for action to process
|
15
|
-
# :a_to:: maximum time action may be worked on before aborting
|
16
|
-
# :logger:: logger to print logging messages to
|
17
|
-
# Creates a new pool
|
18
|
-
def initialize(args={})
|
19
|
-
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
20
|
-
@logger = LogHelper.new(args[:logger])
|
21
|
-
@queue = ActionPool::Queue.new
|
22
|
-
@threads = []
|
23
|
-
@lock = Mutex.new
|
24
|
-
@thread_timeout = args[:t_to] ? args[:t_to] : 0
|
25
|
-
@action_timeout = args[:a_to] ? args[:a_to] : 0
|
26
|
-
@max_threads = args[:max_threads] ? args[:max_threads] : 100
|
27
|
-
@min_threads = args[:min_threads] ? args[:min_threads] : 10
|
28
|
-
@min_threads = @max_threads if @max_threads < @min_threads
|
29
|
-
@respond_to = args[:respond_thread] || ::Thread.current
|
30
|
-
@open = true
|
31
|
-
create_thread
|
32
|
-
end
|
33
|
-
|
34
|
-
# Pool is closed
|
35
|
-
def pool_closed?
|
36
|
-
!@open
|
37
|
-
end
|
38
|
-
|
39
|
-
# Pool is open
|
40
|
-
def pool_open?
|
41
|
-
@open
|
42
|
-
end
|
43
|
-
|
44
|
-
# arg:: :open or :closed
|
45
|
-
# Set pool status
|
46
|
-
def status(arg)
|
47
|
-
@open = arg == :open
|
48
|
-
end
|
49
|
-
|
50
|
-
# force:: force creation of a new thread
|
51
|
-
# Create a new thread for pool. Returns newly created ActionPool::Thread or
|
52
|
-
# nil if pool has reached maximum threads
|
53
|
-
def create_thread(force=false)
|
54
|
-
return if pool_closed?
|
55
|
-
pt = nil
|
56
|
-
@lock.synchronize do
|
57
|
-
if(@threads.size < @max_threads || force)
|
58
|
-
@logger.info('Pool is creating a new thread')
|
59
|
-
(min - size > 0 ? min - size : 1).times do |i|
|
60
|
-
pt = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
|
61
|
-
@threads << pt
|
62
|
-
end
|
63
|
-
else
|
64
|
-
@logger.info('Pool is at maximum size. Not creating new thread')
|
65
|
-
end
|
66
|
-
end
|
67
|
-
return pt
|
68
|
-
end
|
69
|
-
|
70
|
-
# force:: force immediate stop
|
71
|
-
# Stop the pool
|
72
|
-
def shutdown(force=false)
|
73
|
-
args = [:wait]
|
74
|
-
args += [:force] if force
|
75
|
-
@logger.info("Pool is now shutting down #{force ? 'using force' : ''}")
|
76
|
-
@queue.wait_empty
|
77
|
-
while(t = @threads.pop) do
|
78
|
-
t.stop(*args)
|
79
|
-
end
|
80
|
-
nil
|
81
|
-
end
|
82
|
-
|
83
|
-
# action:: proc to be executed or array of [proc, [*args]]
|
84
|
-
# Add a new proc/lambda to be executed (alias for queue)
|
85
|
-
def <<(action)
|
86
|
-
raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
|
87
|
-
case action
|
88
|
-
when Proc
|
89
|
-
queue(action)
|
90
|
-
when Array
|
91
|
-
raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless action.size == 2 and action[0].is_a?(Proc) and action[1].is_a?(Array)
|
92
|
-
queue(action[0], action[1])
|
93
|
-
else
|
94
|
-
raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
|
95
|
-
end
|
96
|
-
nil
|
97
|
-
end
|
98
|
-
|
99
|
-
# action:: proc to be executed
|
100
|
-
# Add a new proc/lambda to be executed
|
101
|
-
def queue(action, *args)
|
102
|
-
raise ArgumentError.new('Expecting block') unless action.is_a?(Proc)
|
103
|
-
@queue << [action, args]
|
104
|
-
::Thread.pass
|
105
|
-
create_thread if @queue.num_waiting < 1 # only start a new thread if we need it
|
106
|
-
end
|
107
|
-
|
108
|
-
# jobs:: Array of proc/lambdas
|
109
|
-
# Will queue a list of jobs into the pool
|
110
|
-
def add_jobs(jobs)
|
111
|
-
raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
|
112
|
-
@queue.pause
|
113
|
-
begin
|
114
|
-
jobs.each do |job|
|
115
|
-
case job
|
116
|
-
when Proc
|
117
|
-
@queue << [job, []]
|
118
|
-
when Array
|
119
|
-
raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
|
120
|
-
@queue << [job.unshift, job]
|
121
|
-
else
|
122
|
-
raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
|
123
|
-
end
|
124
|
-
end
|
125
|
-
ensure
|
126
|
-
create_thread
|
127
|
-
@queue.unpause
|
128
|
-
end
|
129
|
-
true
|
130
|
-
end
|
131
|
-
|
132
|
-
# block:: block to process
|
133
|
-
# Adds a block to be processed
|
134
|
-
def process(*args, &block)
|
135
|
-
queue(block, *args)
|
136
|
-
nil
|
137
|
-
end
|
138
|
-
|
139
|
-
# Current size of pool
|
140
|
-
def size
|
141
|
-
@threads.size
|
142
|
-
end
|
143
|
-
|
144
|
-
# Maximum allowed number of threads
|
145
|
-
def max
|
146
|
-
@max_threads
|
147
|
-
end
|
148
|
-
|
149
|
-
# Minimum allowed number of threads
|
150
|
-
def min
|
151
|
-
@min_threads
|
152
|
-
end
|
153
|
-
|
154
|
-
# m:: new max
|
155
|
-
# Set maximum number of threads
|
156
|
-
def max=(m)
|
157
|
-
m = m.to_i
|
158
|
-
raise ArgumentError.new('Maximum value must be greater than 0') unless m > 0
|
159
|
-
@max_threads = m
|
160
|
-
@min_threads = m if m < @min_threads
|
161
|
-
resize if m < size
|
162
|
-
m
|
163
|
-
end
|
164
|
-
|
165
|
-
# m:: new min
|
166
|
-
# Set minimum number of threads
|
167
|
-
def min=(m)
|
168
|
-
m = m.to_i
|
169
|
-
raise ArgumentError.new("Minimum value must be greater than 0 and less than or equal to maximum (#{max})") unless m > 0 && m <= max
|
170
|
-
@min_threads = m
|
171
|
-
m
|
172
|
-
end
|
173
|
-
|
174
|
-
# t:: ActionPool::Thread to remove
|
175
|
-
# Removes a thread from the pool
|
176
|
-
def remove(t)
|
177
|
-
raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
|
178
|
-
t.stop
|
179
|
-
if(@threads.include?(t))
|
180
|
-
@threads.delete(t)
|
181
|
-
return true
|
182
|
-
else
|
183
|
-
return false
|
184
|
-
end
|
185
|
-
end
|
186
|
-
|
187
|
-
# Maximum number of seconds a thread
|
188
|
-
# is allowed to idle in the pool.
|
189
|
-
# (nil means thread life is infinite)
|
190
|
-
def thread_timeout
|
191
|
-
@thread_timeout
|
192
|
-
end
|
193
|
-
|
194
|
-
# Maximum number of seconds a thread
|
195
|
-
# is allowed to work on a given action
|
196
|
-
# (nil means thread is given unlimited
|
197
|
-
# time to work on action)
|
198
|
-
def action_timeout
|
199
|
-
@action_timeout
|
200
|
-
end
|
201
|
-
|
202
|
-
# t:: timeout in seconds (nil for infinite)
|
203
|
-
# Set maximum allowed time thead may idle in pool
|
204
|
-
def thread_timeout=(t)
|
205
|
-
t = t.to_f
|
206
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
207
|
-
@thread_timeout = t
|
208
|
-
@threads.each{|thread|thread.thread_timeout = t}
|
209
|
-
t
|
210
|
-
end
|
211
|
-
|
212
|
-
# t:: timeout in seconds (nil for infinte)
|
213
|
-
# Set maximum allowed time thread may work
|
214
|
-
# on a given action
|
215
|
-
def action_timeout=(t)
|
216
|
-
t = t.to_f
|
217
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
218
|
-
@action_timeout = t
|
219
|
-
@threads.each{|thread|thread.action_timeout = t}
|
220
|
-
t
|
221
|
-
end
|
222
|
-
|
223
|
-
# Returns the next action to be processed
|
224
|
-
def action
|
225
|
-
@queue.pop
|
226
|
-
end
|
227
|
-
|
228
|
-
# Number of actions in the queue
|
229
|
-
def action_size
|
230
|
-
@queue.size
|
231
|
-
end
|
232
|
-
|
233
|
-
# Flush the thread pool. Mainly used for forcibly resizing
|
234
|
-
# the pool if existing threads have a long thread life waiting
|
235
|
-
# for input.
|
236
|
-
def flush
|
237
|
-
lock = Mutex.new
|
238
|
-
guard = ConditionVariable.new
|
239
|
-
@threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
|
240
|
-
Thread.pass
|
241
|
-
sleep(0.01)
|
242
|
-
lock.synchronize{ guard.broadcast }
|
243
|
-
end
|
244
|
-
|
245
|
-
private
|
246
|
-
|
247
|
-
# Resize the pool
|
248
|
-
def resize
|
249
|
-
@logger.info("Pool is being resized to stated maximum: #{max}")
|
250
|
-
until(size <= max) do
|
251
|
-
t = nil
|
252
|
-
t = @threads.find{|t|t.waiting?}
|
253
|
-
t = @threads.shift unless t
|
254
|
-
t.stop
|
255
|
-
end
|
256
|
-
flush
|
257
|
-
nil
|
258
|
-
end
|
259
|
-
end
|
260
|
-
end
|
data/lib/actionpool/Queue.rb~
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'thread'
|
2
|
-
|
3
|
-
module ActionPool
|
4
|
-
# Adds a little bit extra functionality to the Queue class
|
5
|
-
class Queue < ::Queue
|
6
|
-
# Create a new Queue for the ActionPool::Pool
|
7
|
-
def initialize
|
8
|
-
super
|
9
|
-
@wait = false
|
10
|
-
@pause_lock = Mutex.new
|
11
|
-
@emtpy_lock = Mutex.new
|
12
|
-
@pause_guard = ConditionVariable.new
|
13
|
-
@empty_guard = ConditionVariable.new
|
14
|
-
end
|
15
|
-
# Stop the queue from returning results to requesting
|
16
|
-
# threads. Threads will wait for results until signalled
|
17
|
-
def pause
|
18
|
-
@pause_lock.synchronize{@wait = true}
|
19
|
-
end
|
20
|
-
# Allow the queue to return results. Any threads waiting
|
21
|
-
# will have results given to them.
|
22
|
-
def unpause
|
23
|
-
@pause_lock.synchronize do
|
24
|
-
@wait = false
|
25
|
-
@pause_guard.broadcast
|
26
|
-
end
|
27
|
-
end
|
28
|
-
# Check if queue needs to wait before returning
|
29
|
-
def pop
|
30
|
-
@pause_lock.synchronize do
|
31
|
-
@pause_guard.wait(@pause_lock) if @wait
|
32
|
-
end
|
33
|
-
o = super
|
34
|
-
@empty_lock.synchronize do
|
35
|
-
@empty_guard.broadcast if empty?
|
36
|
-
end
|
37
|
-
return o
|
38
|
-
end
|
39
|
-
# Park a thread here until queue is empty
|
40
|
-
def wait_empty
|
41
|
-
@empty_lock.synchronize do
|
42
|
-
@empty_guard.wait(@empty_lock) if size > 0
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
data/lib/actionpool/Thread.rb~
DELETED
@@ -1,146 +0,0 @@
|
|
1
|
-
require 'timeout'
|
2
|
-
|
3
|
-
module ActionPool
|
4
|
-
# Exception class used for waking up a thread
|
5
|
-
class Wakeup < StandardError
|
6
|
-
end
|
7
|
-
|
8
|
-
class Thread
|
9
|
-
# :pool:: pool thread is associated with
|
10
|
-
# :t_timeout:: max time a thread is allowed to wait for action
|
11
|
-
# :a_timeout:: max time thread is allowed to work
|
12
|
-
# :respond_thread:: thread to send execptions to
|
13
|
-
# :logger:: LogHelper for logging messages
|
14
|
-
# Create a new thread
|
15
|
-
def initialize(args)
|
16
|
-
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
17
|
-
raise ArgumentError.new('ActionPool::Thread requires a pool') unless args[:pool]
|
18
|
-
raise ArgumentError.new('ActionPool::Thread requries thread to respond') unless args[:respond_thread]
|
19
|
-
@pool = args[:pool]
|
20
|
-
@respond_to = args[:respond_thread]
|
21
|
-
@thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
|
22
|
-
@action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
|
23
|
-
@kill = false
|
24
|
-
@logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
|
25
|
-
@lock = Mutex.new
|
26
|
-
@thread = ::Thread.new{ start_thread }
|
27
|
-
end
|
28
|
-
|
29
|
-
# :force:: force the thread to stop
|
30
|
-
# :wait:: wait for the thread to stop
|
31
|
-
# Stop the thread
|
32
|
-
def stop(*args)
|
33
|
-
@kill = true
|
34
|
-
@thread.raise Wakeup.new if args.include?(:force) || waiting?
|
35
|
-
nil
|
36
|
-
end
|
37
|
-
|
38
|
-
# Currently waiting
|
39
|
-
def waiting?
|
40
|
-
@status == :wait
|
41
|
-
end
|
42
|
-
|
43
|
-
# Is the thread still alive
|
44
|
-
def alive?
|
45
|
-
@thread.alive?
|
46
|
-
end
|
47
|
-
|
48
|
-
# Current thread status
|
49
|
-
def status
|
50
|
-
@lock.synchronize{ return @status }
|
51
|
-
end
|
52
|
-
|
53
|
-
# arg:: :wait or :run
|
54
|
-
# Set current status
|
55
|
-
def status(arg)
|
56
|
-
raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
|
57
|
-
@lock.synchronize{ @status = arg }
|
58
|
-
end
|
59
|
-
|
60
|
-
# Seconds thread will wait for input
|
61
|
-
def thread_timeout
|
62
|
-
@thread_timeout
|
63
|
-
end
|
64
|
-
|
65
|
-
# Seconds thread will spend working on a given task
|
66
|
-
def action_timeout
|
67
|
-
@action_timeout
|
68
|
-
end
|
69
|
-
|
70
|
-
# t:: seconds to wait for input (floats allow for values 0 < t < 1)
|
71
|
-
# Set the maximum amount of time to wait for a task
|
72
|
-
def thread_timeout=(t)
|
73
|
-
t = t.to_f
|
74
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
75
|
-
@thread_timeout = t
|
76
|
-
t
|
77
|
-
end
|
78
|
-
|
79
|
-
# t:: seconds to work on a task (floats allow for values 0 < t < 1)
|
80
|
-
# Set the maximum amount of time to work on a given task
|
81
|
-
def action_timeout=(t)
|
82
|
-
t = t.to_f
|
83
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
84
|
-
@action_timeout = t
|
85
|
-
t
|
86
|
-
end
|
87
|
-
|
88
|
-
private
|
89
|
-
|
90
|
-
# Start our thread
|
91
|
-
def start_thread
|
92
|
-
begin
|
93
|
-
@logger.info("New pool thread is starting (#{self})")
|
94
|
-
until(@kill) do
|
95
|
-
status(:wait)
|
96
|
-
begin
|
97
|
-
action = nil
|
98
|
-
if(@pool.size > @pool.min)
|
99
|
-
Timeout::timeout(@thread_timeout) do
|
100
|
-
action = @pool.action
|
101
|
-
end
|
102
|
-
else
|
103
|
-
action = @pool.action
|
104
|
-
end
|
105
|
-
status(:run)
|
106
|
-
run(action[0], action[1]) unless action.nil?
|
107
|
-
status(:wait)
|
108
|
-
rescue Timeout::Error => boom
|
109
|
-
@kill = true
|
110
|
-
rescue Wakeup
|
111
|
-
@logger.info("Thread #{::Thread.current} was woken up.")
|
112
|
-
rescue Exception => boom
|
113
|
-
@logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
|
114
|
-
@respond_to.raise boom
|
115
|
-
end
|
116
|
-
end
|
117
|
-
rescue Wakeup
|
118
|
-
@logger.info("Thread #{::Thread.current} was woken up.")
|
119
|
-
rescue Exception => boom
|
120
|
-
@logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
|
121
|
-
@respond_to.raise boom
|
122
|
-
ensure
|
123
|
-
@logger.info("Pool thread is shutting down (#{self})")
|
124
|
-
@pool.remove(self)
|
125
|
-
@pool.create_threads
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
# action:: task to be run
|
130
|
-
# args:: arguments to be passed to task
|
131
|
-
# Run the task
|
132
|
-
def run(action, args)
|
133
|
-
begin
|
134
|
-
if(@action_timeout > 0)
|
135
|
-
Timeout::timeout(@action_timeout) do
|
136
|
-
action.call(*args.flatten)
|
137
|
-
end
|
138
|
-
else
|
139
|
-
action.call(*args[0])
|
140
|
-
end
|
141
|
-
rescue Timeout::Error => boom
|
142
|
-
@logger.warn("Pool thread reached max execution time for action: #{boom}")
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
data/test.rb
DELETED
data/tests/cases/general.rb~
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
require 'actionpool'
|
2
|
-
require 'test/unit'
|
3
|
-
|
4
|
-
class GeneralPoolTest < Test::Unit::TestCase
|
5
|
-
def setup
|
6
|
-
@pool = ActionPool::Pool.new
|
7
|
-
end
|
8
|
-
def test_numbers
|
9
|
-
assert_equal(10, @pool.size)
|
10
|
-
assert_equal(10, @pool.min)
|
11
|
-
assert_equal(100, @pool.max)
|
12
|
-
assert_equal(0, @pool.action_timeout)
|
13
|
-
assert_equal(0, @pool.thread_timeout)
|
14
|
-
assert_equal(0, @pool.action_size)
|
15
|
-
end
|
16
|
-
def test_output
|
17
|
-
a = 0
|
18
|
-
lock = Mutex.new
|
19
|
-
run = lambda{ lock.synchronize{ a += 1 } }
|
20
|
-
100.times{ @pool << run }
|
21
|
-
@pool.shutdown
|
22
|
-
assert_equal(100, a)
|
23
|
-
a = 0
|
24
|
-
jobs = [].fill(run,0,100)
|
25
|
-
@pool.add_jobs(jobs)
|
26
|
-
@pool.shutdown
|
27
|
-
assert_equal(100, a)
|
28
|
-
@pool.shutdown(true)
|
29
|
-
end
|
30
|
-
def test_args
|
31
|
-
output = nil
|
32
|
-
@pool << [lambda{|x| output = x}, [2]]
|
33
|
-
assert(2, output)
|
34
|
-
@pool.add_jobs([[lambda{|x| output = x}, [3]]])
|
35
|
-
assert(3, output)
|
36
|
-
@pool << [lambda{|x,y| output = x+y}, [1,2]]
|
37
|
-
# assert(3, output)
|
38
|
-
sleep(0.01)
|
39
|
-
output = []
|
40
|
-
@pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
|
41
|
-
# assert(output.include?(2))
|
42
|
-
# assert(output.include?(3))
|
43
|
-
@pool.shutdown(true)
|
44
|
-
end
|
45
|
-
end
|