actionpool 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +6 -0
- data/README.rdoc +10 -1
- data/actionpool.gemspec +2 -1
- data/lib/actionpool.rb +3 -0
- data/lib/actionpool/Pool.rb +40 -27
- data/lib/actionpool/Queue.rb +9 -21
- data/lib/actionpool/Thread.rb +49 -23
- data/tests/cases/general.rb +15 -7
- data/tests/cases/timeouts.rb +25 -12
- metadata +24 -21
- data/lib/actionpool/LogHelper.rb +0 -25
- data/lib/actionpool/Pool.rb~ +0 -260
- data/lib/actionpool/Queue.rb~ +0 -46
- data/lib/actionpool/Thread.rb~ +0 -146
- data/test.rb +0 -7
- data/tests/cases/general.rb~ +0 -45
data/CHANGELOG
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
0.2.3
|
2
|
+
- Better thread destroying when removed normally and forcibly
|
3
|
+
- Flushing threads to ensure proper joining
|
4
|
+
- Fixed splatting for ruby < 1.9
|
5
|
+
- Removed the useless logger wrapper
|
6
|
+
- Use a simple monitor
|
1
7
|
0.2.2
|
2
8
|
- Better thread management within the pool
|
3
9
|
- Restart thread timeouts when modified
|
data/README.rdoc
CHANGED
@@ -4,7 +4,7 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
|
|
4
4
|
|
5
5
|
=== install (easy):
|
6
6
|
|
7
|
-
gem install
|
7
|
+
gem install actionpool
|
8
8
|
|
9
9
|
=== install (less easy):
|
10
10
|
|
@@ -17,6 +17,15 @@ ActionPool is just a simple thread pool. It allows for various constraints and r
|
|
17
17
|
|
18
18
|
{rip}[http://hellorip.com/about.html] makes it easy to install directly from a github repository.
|
19
19
|
|
20
|
+
=== Testing
|
21
|
+
|
22
|
+
ActionPool is currently tested on:
|
23
|
+
|
24
|
+
* Ruby 1.8.6-p383
|
25
|
+
* Ruby 1.8.7-p248
|
26
|
+
* Ruby 1.9.1-p376
|
27
|
+
* JRuby 1.4.0
|
28
|
+
|
20
29
|
== Documentation
|
21
30
|
|
22
31
|
{rdocs}[http://allgems.ruby-forum.com/gems/ActionPool/]
|
data/actionpool.gemspec
CHANGED
@@ -2,7 +2,7 @@ spec = Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'actionpool'
|
3
3
|
s.author = %q(spox)
|
4
4
|
s.email = %q(spox@modspox.com)
|
5
|
-
s.version = '0.2.
|
5
|
+
s.version = '0.2.3'
|
6
6
|
s.summary = %q(Thread Pool)
|
7
7
|
s.platform = Gem::Platform::RUBY
|
8
8
|
s.files = Dir['**/*']
|
@@ -10,6 +10,7 @@ spec = Gem::Specification.new do |s|
|
|
10
10
|
s.extra_rdoc_files = %w(README.rdoc CHANGELOG)
|
11
11
|
s.require_paths = %w(lib)
|
12
12
|
s.required_ruby_version = '>= 1.8.6'
|
13
|
+
s.add_dependency 'splib', '~> 1.4'
|
13
14
|
s.homepage = %q(http://github.com/spox/actionpool)
|
14
15
|
s.description = "The ActionPool is an easy to use thread pool for ruby."
|
15
16
|
end
|
data/lib/actionpool.rb
CHANGED
data/lib/actionpool/Pool.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'actionpool/Thread'
|
2
2
|
require 'actionpool/Queue'
|
3
|
-
require '
|
3
|
+
require 'logger'
|
4
4
|
require 'thread'
|
5
5
|
|
6
6
|
module ActionPool
|
@@ -17,10 +17,10 @@ module ActionPool
|
|
17
17
|
# Creates a new pool
|
18
18
|
def initialize(args={})
|
19
19
|
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
20
|
-
@logger =
|
20
|
+
@logger = args[:logger] && args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
|
21
21
|
@queue = ActionPool::Queue.new
|
22
22
|
@threads = []
|
23
|
-
@lock =
|
23
|
+
@lock = Splib::Monitor.new
|
24
24
|
@thread_timeout = args[:t_to] ? args[:t_to] : 0
|
25
25
|
@action_timeout = args[:a_to] ? args[:a_to] : 0
|
26
26
|
@max_threads = args[:max_threads] ? args[:max_threads] : 100
|
@@ -48,36 +48,44 @@ module ActionPool
|
|
48
48
|
fill_pool if @open
|
49
49
|
end
|
50
50
|
|
51
|
-
# args::
|
51
|
+
# args:: :force forces a new thread.
|
52
|
+
# :nowait will create a thread if threads are waiting
|
52
53
|
# Create a new thread for pool.
|
53
|
-
# Returns newly created thread
|
54
|
+
# Returns newly created thread or nil if pool is at maximum size
|
54
55
|
def create_thread(*args)
|
55
56
|
return if pool_closed?
|
56
57
|
thread = nil
|
57
58
|
@lock.synchronize do
|
58
59
|
if(((size == working || args.include?(:nowait)) && @threads.size < @max_threads) || args.include?(:force))
|
59
|
-
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
|
60
|
+
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout,
|
61
|
+
:t_timeout => @thread_timeout, :logger => @logger, :autostart => false)
|
60
62
|
@threads << thread
|
61
63
|
end
|
62
64
|
end
|
63
|
-
|
65
|
+
thread.start if thread
|
66
|
+
thread
|
64
67
|
end
|
65
68
|
|
66
69
|
# Fills the pool with the minimum number of threads
|
67
70
|
# Returns array of created threads
|
68
71
|
def fill_pool
|
69
72
|
threads = []
|
70
|
-
@
|
71
|
-
|
72
|
-
|
73
|
-
required
|
74
|
-
|
75
|
-
|
76
|
-
|
73
|
+
if(@open)
|
74
|
+
@lock.synchronize do
|
75
|
+
required = min - size
|
76
|
+
if(required > 0)
|
77
|
+
required.times do
|
78
|
+
thread = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to,
|
79
|
+
:a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger,
|
80
|
+
:autostart => false)
|
81
|
+
@threads << thread
|
82
|
+
threads << thread
|
83
|
+
end
|
77
84
|
end
|
78
85
|
end
|
79
86
|
end
|
80
|
-
|
87
|
+
threads.each{|t|t.start}
|
88
|
+
threads
|
81
89
|
end
|
82
90
|
|
83
91
|
# force:: force immediate stop
|
@@ -92,6 +100,10 @@ module ActionPool
|
|
92
100
|
while(t = @threads.pop) do
|
93
101
|
t.stop(*args)
|
94
102
|
end
|
103
|
+
unless(force)
|
104
|
+
flush
|
105
|
+
@threads.each{|t|t.join}
|
106
|
+
end
|
95
107
|
nil
|
96
108
|
end
|
97
109
|
|
@@ -193,12 +205,10 @@ module ActionPool
|
|
193
205
|
def remove(t)
|
194
206
|
raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
|
195
207
|
t.stop
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
return false
|
201
|
-
end
|
208
|
+
del = @threads.include?(t)
|
209
|
+
@threads.delete(t) if del
|
210
|
+
fill_pool
|
211
|
+
del
|
202
212
|
end
|
203
213
|
|
204
214
|
# Maximum number of seconds a thread
|
@@ -251,17 +261,20 @@ module ActionPool
|
|
251
261
|
# the pool if existing threads have a long thread life waiting
|
252
262
|
# for input.
|
253
263
|
def flush
|
254
|
-
|
255
|
-
|
256
|
-
@
|
257
|
-
Thread.pass
|
264
|
+
mon = Splib::Monitor.new
|
265
|
+
@threads.size.times{ queue{ mon.wait } }
|
266
|
+
@queue.wait_empty
|
258
267
|
sleep(0.01)
|
259
|
-
|
268
|
+
mon.broadcast
|
260
269
|
end
|
261
270
|
|
262
271
|
# Returns current number of threads in the pool working
|
263
272
|
def working
|
264
|
-
@threads.
|
273
|
+
@threads.select{|t|t.running?}.size
|
274
|
+
end
|
275
|
+
|
276
|
+
def thread_stats
|
277
|
+
@threads.map{|t|[t.object_id,t.status]}
|
265
278
|
end
|
266
279
|
|
267
280
|
private
|
data/lib/actionpool/Queue.rb
CHANGED
@@ -7,47 +7,35 @@ module ActionPool
|
|
7
7
|
def initialize
|
8
8
|
super
|
9
9
|
@wait = false
|
10
|
-
@
|
11
|
-
@
|
12
|
-
@pause_guard = ConditionVariable.new
|
13
|
-
@empty_guard = ConditionVariable.new
|
10
|
+
@pause_guard = Splib::Monitor.new
|
11
|
+
@empty_guard = Splib::Monitor.new
|
14
12
|
end
|
15
13
|
# Stop the queue from returning results to requesting
|
16
14
|
# threads. Threads will wait for results until signalled
|
17
15
|
def pause
|
18
|
-
@
|
16
|
+
@wait = true
|
19
17
|
end
|
20
18
|
# Allow the queue to return results. Any threads waiting
|
21
19
|
# will have results given to them.
|
22
20
|
def unpause
|
23
|
-
@
|
24
|
-
|
25
|
-
@pause_guard.broadcast
|
26
|
-
end
|
21
|
+
@wait = false
|
22
|
+
@pause_guard.broadcast
|
27
23
|
end
|
28
24
|
# Check if queue needs to wait before returning
|
29
25
|
def pop
|
30
|
-
@
|
31
|
-
@pause_guard.wait(@pause_lock) if @wait
|
32
|
-
end
|
26
|
+
@pause_guard.wait_while{ @wait }
|
33
27
|
o = super
|
34
|
-
@
|
35
|
-
@empty_guard.broadcast if empty?
|
36
|
-
end
|
28
|
+
@empty_guard.broadcast if empty?
|
37
29
|
return o
|
38
30
|
end
|
39
31
|
# Clear queue
|
40
32
|
def clear
|
41
33
|
super
|
42
|
-
@
|
43
|
-
@empty_guard.broadcast
|
44
|
-
end
|
34
|
+
@empty_guard.broadcast
|
45
35
|
end
|
46
36
|
# Park a thread here until queue is empty
|
47
37
|
def wait_empty
|
48
|
-
@
|
49
|
-
@empty_guard.wait(@empty_lock) if size > 0
|
50
|
-
end
|
38
|
+
@empty_guard.wait_while{ size > 0 }
|
51
39
|
end
|
52
40
|
end
|
53
41
|
end
|
data/lib/actionpool/Thread.rb
CHANGED
@@ -13,6 +13,7 @@ module ActionPool
|
|
13
13
|
# :a_timeout:: max time thread is allowed to work
|
14
14
|
# :respond_thread:: thread to send execptions to
|
15
15
|
# :logger:: LogHelper for logging messages
|
16
|
+
# :autostart:: Automatically start the thread
|
16
17
|
# Create a new thread
|
17
18
|
def initialize(args)
|
18
19
|
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
@@ -22,24 +23,45 @@ module ActionPool
|
|
22
23
|
@respond_to = args[:respond_thread]
|
23
24
|
@thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
|
24
25
|
@action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
|
26
|
+
args[:autostart] = true unless args.has_key?(:autostart)
|
25
27
|
@kill = false
|
26
|
-
@logger = args[:logger].is_a?(
|
27
|
-
@lock =
|
28
|
-
@
|
28
|
+
@logger = args[:logger].is_a?(Logger) ? args[:logger] : Logger.new(nil)
|
29
|
+
@lock = Splib::Monitor.new
|
30
|
+
@action = nil
|
31
|
+
@thread = args[:autostart] ? ::Thread.new{ start_thread } : nil
|
29
32
|
end
|
30
33
|
|
34
|
+
def start
|
35
|
+
@thread = ::Thread.new{ start_thread } if @thread.nil?
|
36
|
+
end
|
37
|
+
|
31
38
|
# :force:: force the thread to stop
|
32
39
|
# :wait:: wait for the thread to stop
|
33
40
|
# Stop the thread
|
34
41
|
def stop(*args)
|
35
42
|
@kill = true
|
36
|
-
|
43
|
+
if(args.include?(:force) || waiting?)
|
44
|
+
begin
|
45
|
+
@thread.raise Wakeup.new
|
46
|
+
rescue Wakeup
|
47
|
+
#ignore since we are the caller
|
48
|
+
end
|
49
|
+
sleep(0.01)
|
50
|
+
@thread.kill if @thread.alive?
|
51
|
+
end
|
37
52
|
nil
|
38
53
|
end
|
39
54
|
|
40
55
|
# Currently waiting
|
41
56
|
def waiting?
|
42
|
-
@
|
57
|
+
@action.nil?
|
58
|
+
# @status == :wait
|
59
|
+
end
|
60
|
+
|
61
|
+
# Currently running
|
62
|
+
def running?
|
63
|
+
!@action.nil?
|
64
|
+
# @status == :run
|
43
65
|
end
|
44
66
|
|
45
67
|
# Is the thread still alive
|
@@ -49,14 +71,21 @@ module ActionPool
|
|
49
71
|
|
50
72
|
# Current thread status
|
51
73
|
def status
|
52
|
-
@
|
74
|
+
@action
|
53
75
|
end
|
54
|
-
|
55
|
-
#
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
76
|
+
|
77
|
+
# Join internal thread
|
78
|
+
def join
|
79
|
+
@thread.join(@action_timeout)
|
80
|
+
if(@thread.alive?)
|
81
|
+
@thread.kill
|
82
|
+
@thread.join
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Kill internal thread
|
87
|
+
def kill
|
88
|
+
@thread.kill
|
60
89
|
end
|
61
90
|
|
62
91
|
# Seconds thread will wait for input
|
@@ -96,20 +125,17 @@ module ActionPool
|
|
96
125
|
begin
|
97
126
|
@logger.info("New pool thread is starting (#{self})")
|
98
127
|
until(@kill) do
|
99
|
-
status(:wait)
|
100
128
|
begin
|
101
|
-
action = nil
|
129
|
+
@action = nil
|
102
130
|
if(@pool.size > @pool.min && !@thread_timeout.zero?)
|
103
131
|
Timeout::timeout(@thread_timeout) do
|
104
|
-
action = @pool.action
|
132
|
+
@action = @pool.action
|
105
133
|
end
|
106
134
|
else
|
107
|
-
action = @pool.action
|
135
|
+
@action = @pool.action
|
108
136
|
end
|
109
|
-
|
110
|
-
|
111
|
-
status(:wait)
|
112
|
-
rescue Timeout::Error => boom
|
137
|
+
run(@action[0], @action[1]) unless @action.nil?
|
138
|
+
rescue Timeout::Error
|
113
139
|
@kill = true
|
114
140
|
rescue Wakeup
|
115
141
|
@logger.info("Thread #{::Thread.current} was woken up.")
|
@@ -131,7 +157,6 @@ module ActionPool
|
|
131
157
|
ensure
|
132
158
|
@logger.info("Pool thread is shutting down (#{self})")
|
133
159
|
@pool.remove(self)
|
134
|
-
@pool.create_thread
|
135
160
|
end
|
136
161
|
end
|
137
162
|
|
@@ -139,13 +164,14 @@ module ActionPool
|
|
139
164
|
# args:: arguments to be passed to task
|
140
165
|
# Run the task
|
141
166
|
def run(action, args)
|
167
|
+
args = args.respond_to?(:fixed_flatten) ? args.fixed_flatten(1) : args.flatten(1)
|
142
168
|
begin
|
143
169
|
unless(@action_timeout.zero?)
|
144
170
|
Timeout::timeout(@action_timeout) do
|
145
|
-
action.call(*args
|
171
|
+
action.call(*args)
|
146
172
|
end
|
147
173
|
else
|
148
|
-
action.call(*args
|
174
|
+
action.call(*args)
|
149
175
|
end
|
150
176
|
rescue Timeout::Error => boom
|
151
177
|
@logger.warn("Pool thread reached max execution time for action: #{boom}")
|
data/tests/cases/general.rb
CHANGED
@@ -28,6 +28,7 @@ class GeneralPoolTest < Test::Unit::TestCase
|
|
28
28
|
jobs = [].fill(run,0,100)
|
29
29
|
@pool.add_jobs(jobs)
|
30
30
|
@pool.shutdown
|
31
|
+
sleep(0.01)
|
31
32
|
assert_equal(100, a)
|
32
33
|
@pool.shutdown(true)
|
33
34
|
end
|
@@ -36,17 +37,24 @@ class GeneralPoolTest < Test::Unit::TestCase
|
|
36
37
|
output = nil
|
37
38
|
@pool << [lambda{|x| output = x}, [2]]
|
38
39
|
assert(2, output)
|
40
|
+
output = nil
|
39
41
|
@pool.add_jobs([[lambda{|x| output = x}, [3]]])
|
40
42
|
assert(3, output)
|
43
|
+
output = nil
|
41
44
|
@pool << [lambda{|x,y| output = x+y}, [1,2]]
|
42
45
|
assert(3, output)
|
43
|
-
output =
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
assert(
|
46
|
+
output = nil
|
47
|
+
arr = []
|
48
|
+
@pool.add_jobs([[lambda{|x,y| arr << x + y}, [1,1]], [lambda{|x| arr << x}, [3]]])
|
49
|
+
::Thread.pass
|
50
|
+
sleep(0.01)
|
51
|
+
assert(arr.include?(2))
|
52
|
+
assert(arr.include?(3))
|
53
|
+
arr.clear
|
54
|
+
@pool << [lambda{|x,y| arr = [x,y]}, ['test', [1,2]]]
|
55
|
+
sleep(0.01)
|
56
|
+
assert_equal('test', arr[0])
|
57
|
+
assert(arr[1].is_a?(Array))
|
50
58
|
@pool.shutdown(true)
|
51
59
|
end
|
52
60
|
end
|
data/tests/cases/timeouts.rb
CHANGED
@@ -9,26 +9,39 @@ class TimeoutPoolTest < Test::Unit::TestCase
|
|
9
9
|
@pool.shutdown(true)
|
10
10
|
end
|
11
11
|
def test_actiontimeout
|
12
|
-
@pool.action_timeout = 0.
|
12
|
+
@pool.action_timeout = 0.25
|
13
13
|
assert_equal(10, @pool.size)
|
14
14
|
stop = false
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
output = []
|
16
|
+
20.times do
|
17
|
+
@pool.process do
|
18
|
+
until(stop) do
|
19
|
+
output << 1
|
20
|
+
sleep(0.1)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
assert(@pool.working >= 10)
|
25
|
+
assert_equal(20, output.size)
|
26
|
+
sleep(0.11)
|
18
27
|
stop = true
|
19
|
-
sleep(0.
|
20
|
-
|
28
|
+
sleep(0.1)
|
29
|
+
assert_equal(0, @pool.working)
|
30
|
+
assert_equal(40, output.size)
|
21
31
|
@pool.shutdown(true)
|
22
32
|
end
|
23
33
|
def test_threadtimeout
|
24
|
-
@pool.thread_timeout = 0.
|
34
|
+
@pool.thread_timeout = 0.05
|
25
35
|
assert_equal(10, @pool.size)
|
26
|
-
|
27
|
-
|
28
|
-
|
36
|
+
t = [].fill(lambda{
|
37
|
+
begin
|
38
|
+
sleep(0.1)
|
39
|
+
rescue
|
40
|
+
end }, 0, 20)
|
41
|
+
@pool.add_jobs(t)
|
29
42
|
::Thread.pass
|
30
|
-
|
31
|
-
|
43
|
+
sleep(0.01)
|
44
|
+
assert(@pool.size >= 20)
|
32
45
|
::Thread.pass
|
33
46
|
sleep(0.1)
|
34
47
|
assert(10, @pool.size)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: actionpool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- spox
|
@@ -9,10 +9,19 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-01-
|
12
|
+
date: 2010-01-13 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
|
-
dependencies:
|
15
|
-
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: splib
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ~>
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "1.4"
|
24
|
+
version:
|
16
25
|
description: The ActionPool is an easy to use thread pool for ruby.
|
17
26
|
email: spox@modspox.com
|
18
27
|
executables: []
|
@@ -23,30 +32,24 @@ extra_rdoc_files:
|
|
23
32
|
- README.rdoc
|
24
33
|
- CHANGELOG
|
25
34
|
files:
|
26
|
-
-
|
27
|
-
-
|
28
|
-
- lib/actionpool/Pool.rb
|
29
|
-
- lib/actionpool/Thread.rb
|
30
|
-
- lib/actionpool/Queue.rb~
|
31
|
-
- lib/actionpool/Queue.rb
|
32
|
-
- lib/actionpool/Thread.rb~
|
33
|
-
- lib/actionpool.rb
|
34
|
-
- tests/cases/shutdown.rb
|
35
|
-
- tests/cases/nogrow.rb
|
35
|
+
- Rakefile
|
36
|
+
- actionpool.gemspec
|
36
37
|
- tests/cases/general.rb
|
37
|
-
- tests/cases/thread.rb
|
38
|
-
- tests/cases/grow.rb
|
39
|
-
- tests/cases/general.rb~
|
40
|
-
- tests/cases/timeouts.rb
|
41
38
|
- tests/cases/resize.rb
|
39
|
+
- tests/cases/nogrow.rb
|
40
|
+
- tests/cases/timeouts.rb
|
41
|
+
- tests/cases/thread.rb
|
42
|
+
- tests/cases/shutdown.rb
|
42
43
|
- tests/cases/queue.rb
|
44
|
+
- tests/cases/grow.rb
|
43
45
|
- tests/run_tests.rb
|
44
|
-
-
|
46
|
+
- lib/actionpool/Pool.rb
|
47
|
+
- lib/actionpool/Thread.rb
|
48
|
+
- lib/actionpool/Queue.rb
|
49
|
+
- lib/actionpool.rb
|
45
50
|
- CHANGELOG
|
46
51
|
- LICENSE
|
47
52
|
- README.rdoc
|
48
|
-
- actionpool.gemspec
|
49
|
-
- test.rb
|
50
53
|
has_rdoc: true
|
51
54
|
homepage: http://github.com/spox/actionpool
|
52
55
|
licenses: []
|
data/lib/actionpool/LogHelper.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
module ActionPool
|
2
|
-
class LogHelper
|
3
|
-
|
4
|
-
def initialize(logger=nil)
|
5
|
-
require 'logger'
|
6
|
-
@logger = logger
|
7
|
-
end
|
8
|
-
|
9
|
-
def info(m)
|
10
|
-
@logger.info(m) unless @logger.nil?
|
11
|
-
end
|
12
|
-
|
13
|
-
def warn(m)
|
14
|
-
@logger.warn(m) unless @logger.nil?
|
15
|
-
end
|
16
|
-
|
17
|
-
def fatal(m)
|
18
|
-
@logger.fatal(m) unless @logger.nil?
|
19
|
-
end
|
20
|
-
|
21
|
-
def error(m)
|
22
|
-
@logger.error(m) unless @logger.nil?
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
data/lib/actionpool/Pool.rb~
DELETED
@@ -1,260 +0,0 @@
|
|
1
|
-
require 'actionpool/Thread'
|
2
|
-
require 'actionpool/Queue'
|
3
|
-
require 'actionpool/LogHelper'
|
4
|
-
require 'thread'
|
5
|
-
|
6
|
-
module ActionPool
|
7
|
-
# Raised when pool is closed
|
8
|
-
class PoolClosed < StandardError
|
9
|
-
end
|
10
|
-
class Pool
|
11
|
-
|
12
|
-
# :min_threads:: minimum number of threads in pool
|
13
|
-
# :max_threads:: maximum number of threads in pool
|
14
|
-
# :t_to:: thread timeout waiting for action to process
|
15
|
-
# :a_to:: maximum time action may be worked on before aborting
|
16
|
-
# :logger:: logger to print logging messages to
|
17
|
-
# Creates a new pool
|
18
|
-
def initialize(args={})
|
19
|
-
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
20
|
-
@logger = LogHelper.new(args[:logger])
|
21
|
-
@queue = ActionPool::Queue.new
|
22
|
-
@threads = []
|
23
|
-
@lock = Mutex.new
|
24
|
-
@thread_timeout = args[:t_to] ? args[:t_to] : 0
|
25
|
-
@action_timeout = args[:a_to] ? args[:a_to] : 0
|
26
|
-
@max_threads = args[:max_threads] ? args[:max_threads] : 100
|
27
|
-
@min_threads = args[:min_threads] ? args[:min_threads] : 10
|
28
|
-
@min_threads = @max_threads if @max_threads < @min_threads
|
29
|
-
@respond_to = args[:respond_thread] || ::Thread.current
|
30
|
-
@open = true
|
31
|
-
create_thread
|
32
|
-
end
|
33
|
-
|
34
|
-
# Pool is closed
|
35
|
-
def pool_closed?
|
36
|
-
!@open
|
37
|
-
end
|
38
|
-
|
39
|
-
# Pool is open
|
40
|
-
def pool_open?
|
41
|
-
@open
|
42
|
-
end
|
43
|
-
|
44
|
-
# arg:: :open or :closed
|
45
|
-
# Set pool status
|
46
|
-
def status(arg)
|
47
|
-
@open = arg == :open
|
48
|
-
end
|
49
|
-
|
50
|
-
# force:: force creation of a new thread
|
51
|
-
# Create a new thread for pool. Returns newly created ActionPool::Thread or
|
52
|
-
# nil if pool has reached maximum threads
|
53
|
-
def create_thread(force=false)
|
54
|
-
return if pool_closed?
|
55
|
-
pt = nil
|
56
|
-
@lock.synchronize do
|
57
|
-
if(@threads.size < @max_threads || force)
|
58
|
-
@logger.info('Pool is creating a new thread')
|
59
|
-
(min - size > 0 ? min - size : 1).times do |i|
|
60
|
-
pt = ActionPool::Thread.new(:pool => self, :respond_thread => @respond_to, :a_timeout => @action_timeout, :t_timeout => @thread_timeout, :logger => @logger)
|
61
|
-
@threads << pt
|
62
|
-
end
|
63
|
-
else
|
64
|
-
@logger.info('Pool is at maximum size. Not creating new thread')
|
65
|
-
end
|
66
|
-
end
|
67
|
-
return pt
|
68
|
-
end
|
69
|
-
|
70
|
-
# force:: force immediate stop
|
71
|
-
# Stop the pool
|
72
|
-
def shutdown(force=false)
|
73
|
-
args = [:wait]
|
74
|
-
args += [:force] if force
|
75
|
-
@logger.info("Pool is now shutting down #{force ? 'using force' : ''}")
|
76
|
-
@queue.wait_empty
|
77
|
-
while(t = @threads.pop) do
|
78
|
-
t.stop(*args)
|
79
|
-
end
|
80
|
-
nil
|
81
|
-
end
|
82
|
-
|
83
|
-
# action:: proc to be executed or array of [proc, [*args]]
|
84
|
-
# Add a new proc/lambda to be executed (alias for queue)
|
85
|
-
def <<(action)
|
86
|
-
raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
|
87
|
-
case action
|
88
|
-
when Proc
|
89
|
-
queue(action)
|
90
|
-
when Array
|
91
|
-
raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless action.size == 2 and action[0].is_a?(Proc) and action[1].is_a?(Array)
|
92
|
-
queue(action[0], action[1])
|
93
|
-
else
|
94
|
-
raise ArgumentError.new('Actions to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
|
95
|
-
end
|
96
|
-
nil
|
97
|
-
end
|
98
|
-
|
99
|
-
# action:: proc to be executed
|
100
|
-
# Add a new proc/lambda to be executed
|
101
|
-
def queue(action, *args)
|
102
|
-
raise ArgumentError.new('Expecting block') unless action.is_a?(Proc)
|
103
|
-
@queue << [action, args]
|
104
|
-
::Thread.pass
|
105
|
-
create_thread if @queue.num_waiting < 1 # only start a new thread if we need it
|
106
|
-
end
|
107
|
-
|
108
|
-
# jobs:: Array of proc/lambdas
|
109
|
-
# Will queue a list of jobs into the pool
|
110
|
-
def add_jobs(jobs)
|
111
|
-
raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
|
112
|
-
@queue.pause
|
113
|
-
begin
|
114
|
-
jobs.each do |job|
|
115
|
-
case job
|
116
|
-
when Proc
|
117
|
-
@queue << [job, []]
|
118
|
-
when Array
|
119
|
-
raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
|
120
|
-
@queue << [job.unshift, job]
|
121
|
-
else
|
122
|
-
raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
|
123
|
-
end
|
124
|
-
end
|
125
|
-
ensure
|
126
|
-
create_thread
|
127
|
-
@queue.unpause
|
128
|
-
end
|
129
|
-
true
|
130
|
-
end
|
131
|
-
|
132
|
-
# block:: block to process
|
133
|
-
# Adds a block to be processed
|
134
|
-
def process(*args, &block)
|
135
|
-
queue(block, *args)
|
136
|
-
nil
|
137
|
-
end
|
138
|
-
|
139
|
-
# Current size of pool
|
140
|
-
def size
|
141
|
-
@threads.size
|
142
|
-
end
|
143
|
-
|
144
|
-
# Maximum allowed number of threads
|
145
|
-
def max
|
146
|
-
@max_threads
|
147
|
-
end
|
148
|
-
|
149
|
-
# Minimum allowed number of threads
|
150
|
-
def min
|
151
|
-
@min_threads
|
152
|
-
end
|
153
|
-
|
154
|
-
# m:: new max
|
155
|
-
# Set maximum number of threads
|
156
|
-
def max=(m)
|
157
|
-
m = m.to_i
|
158
|
-
raise ArgumentError.new('Maximum value must be greater than 0') unless m > 0
|
159
|
-
@max_threads = m
|
160
|
-
@min_threads = m if m < @min_threads
|
161
|
-
resize if m < size
|
162
|
-
m
|
163
|
-
end
|
164
|
-
|
165
|
-
# m:: new min
|
166
|
-
# Set minimum number of threads
|
167
|
-
def min=(m)
|
168
|
-
m = m.to_i
|
169
|
-
raise ArgumentError.new("Minimum value must be greater than 0 and less than or equal to maximum (#{max})") unless m > 0 && m <= max
|
170
|
-
@min_threads = m
|
171
|
-
m
|
172
|
-
end
|
173
|
-
|
174
|
-
# t:: ActionPool::Thread to remove
|
175
|
-
# Removes a thread from the pool
|
176
|
-
def remove(t)
|
177
|
-
raise ArgumentError.new('Expecting an ActionPool::Thread object') unless t.is_a?(ActionPool::Thread)
|
178
|
-
t.stop
|
179
|
-
if(@threads.include?(t))
|
180
|
-
@threads.delete(t)
|
181
|
-
return true
|
182
|
-
else
|
183
|
-
return false
|
184
|
-
end
|
185
|
-
end
|
186
|
-
|
187
|
-
# Maximum number of seconds a thread
|
188
|
-
# is allowed to idle in the pool.
|
189
|
-
# (nil means thread life is infinite)
|
190
|
-
def thread_timeout
|
191
|
-
@thread_timeout
|
192
|
-
end
|
193
|
-
|
194
|
-
# Maximum number of seconds a thread
|
195
|
-
# is allowed to work on a given action
|
196
|
-
# (nil means thread is given unlimited
|
197
|
-
# time to work on action)
|
198
|
-
def action_timeout
|
199
|
-
@action_timeout
|
200
|
-
end
|
201
|
-
|
202
|
-
# t:: timeout in seconds (nil for infinite)
|
203
|
-
# Set maximum allowed time thead may idle in pool
|
204
|
-
def thread_timeout=(t)
|
205
|
-
t = t.to_f
|
206
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
207
|
-
@thread_timeout = t
|
208
|
-
@threads.each{|thread|thread.thread_timeout = t}
|
209
|
-
t
|
210
|
-
end
|
211
|
-
|
212
|
-
# t:: timeout in seconds (nil for infinte)
|
213
|
-
# Set maximum allowed time thread may work
|
214
|
-
# on a given action
|
215
|
-
def action_timeout=(t)
|
216
|
-
t = t.to_f
|
217
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
218
|
-
@action_timeout = t
|
219
|
-
@threads.each{|thread|thread.action_timeout = t}
|
220
|
-
t
|
221
|
-
end
|
222
|
-
|
223
|
-
# Returns the next action to be processed
|
224
|
-
def action
|
225
|
-
@queue.pop
|
226
|
-
end
|
227
|
-
|
228
|
-
# Number of actions in the queue
|
229
|
-
def action_size
|
230
|
-
@queue.size
|
231
|
-
end
|
232
|
-
|
233
|
-
# Flush the thread pool. Mainly used for forcibly resizing
|
234
|
-
# the pool if existing threads have a long thread life waiting
|
235
|
-
# for input.
|
236
|
-
def flush
|
237
|
-
lock = Mutex.new
|
238
|
-
guard = ConditionVariable.new
|
239
|
-
@threads.size.times{ queue{ lock.synchronize{ guard.wait(lock) } } }
|
240
|
-
Thread.pass
|
241
|
-
sleep(0.01)
|
242
|
-
lock.synchronize{ guard.broadcast }
|
243
|
-
end
|
244
|
-
|
245
|
-
private
|
246
|
-
|
247
|
-
# Resize the pool
|
248
|
-
def resize
|
249
|
-
@logger.info("Pool is being resized to stated maximum: #{max}")
|
250
|
-
until(size <= max) do
|
251
|
-
t = nil
|
252
|
-
t = @threads.find{|t|t.waiting?}
|
253
|
-
t = @threads.shift unless t
|
254
|
-
t.stop
|
255
|
-
end
|
256
|
-
flush
|
257
|
-
nil
|
258
|
-
end
|
259
|
-
end
|
260
|
-
end
|
data/lib/actionpool/Queue.rb~
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'thread'
|
2
|
-
|
3
|
-
module ActionPool
|
4
|
-
# Adds a little bit extra functionality to the Queue class
|
5
|
-
class Queue < ::Queue
|
6
|
-
# Create a new Queue for the ActionPool::Pool
|
7
|
-
def initialize
|
8
|
-
super
|
9
|
-
@wait = false
|
10
|
-
@pause_lock = Mutex.new
|
11
|
-
@emtpy_lock = Mutex.new
|
12
|
-
@pause_guard = ConditionVariable.new
|
13
|
-
@empty_guard = ConditionVariable.new
|
14
|
-
end
|
15
|
-
# Stop the queue from returning results to requesting
|
16
|
-
# threads. Threads will wait for results until signalled
|
17
|
-
def pause
|
18
|
-
@pause_lock.synchronize{@wait = true}
|
19
|
-
end
|
20
|
-
# Allow the queue to return results. Any threads waiting
|
21
|
-
# will have results given to them.
|
22
|
-
def unpause
|
23
|
-
@pause_lock.synchronize do
|
24
|
-
@wait = false
|
25
|
-
@pause_guard.broadcast
|
26
|
-
end
|
27
|
-
end
|
28
|
-
# Check if queue needs to wait before returning
|
29
|
-
def pop
|
30
|
-
@pause_lock.synchronize do
|
31
|
-
@pause_guard.wait(@pause_lock) if @wait
|
32
|
-
end
|
33
|
-
o = super
|
34
|
-
@empty_lock.synchronize do
|
35
|
-
@empty_guard.broadcast if empty?
|
36
|
-
end
|
37
|
-
return o
|
38
|
-
end
|
39
|
-
# Park a thread here until queue is empty
|
40
|
-
def wait_empty
|
41
|
-
@empty_lock.synchronize do
|
42
|
-
@empty_guard.wait(@empty_lock) if size > 0
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
data/lib/actionpool/Thread.rb~
DELETED
@@ -1,146 +0,0 @@
|
|
1
|
-
require 'timeout'
|
2
|
-
|
3
|
-
module ActionPool
|
4
|
-
# Exception class used for waking up a thread
|
5
|
-
class Wakeup < StandardError
|
6
|
-
end
|
7
|
-
|
8
|
-
class Thread
|
9
|
-
# :pool:: pool thread is associated with
|
10
|
-
# :t_timeout:: max time a thread is allowed to wait for action
|
11
|
-
# :a_timeout:: max time thread is allowed to work
|
12
|
-
# :respond_thread:: thread to send execptions to
|
13
|
-
# :logger:: LogHelper for logging messages
|
14
|
-
# Create a new thread
|
15
|
-
def initialize(args)
|
16
|
-
raise ArgumentError.new('Hash required for initialization') unless args.is_a?(Hash)
|
17
|
-
raise ArgumentError.new('ActionPool::Thread requires a pool') unless args[:pool]
|
18
|
-
raise ArgumentError.new('ActionPool::Thread requries thread to respond') unless args[:respond_thread]
|
19
|
-
@pool = args[:pool]
|
20
|
-
@respond_to = args[:respond_thread]
|
21
|
-
@thread_timeout = args[:t_timeout] ? args[:t_timeout].to_f : 0
|
22
|
-
@action_timeout = args[:a_timeout] ? args[:a_timeout].to_f : 0
|
23
|
-
@kill = false
|
24
|
-
@logger = args[:logger].is_a?(LogHelper) ? args[:logger] : LogHelper.new(args[:logger])
|
25
|
-
@lock = Mutex.new
|
26
|
-
@thread = ::Thread.new{ start_thread }
|
27
|
-
end
|
28
|
-
|
29
|
-
# :force:: force the thread to stop
|
30
|
-
# :wait:: wait for the thread to stop
|
31
|
-
# Stop the thread
|
32
|
-
def stop(*args)
|
33
|
-
@kill = true
|
34
|
-
@thread.raise Wakeup.new if args.include?(:force) || waiting?
|
35
|
-
nil
|
36
|
-
end
|
37
|
-
|
38
|
-
# Currently waiting
|
39
|
-
def waiting?
|
40
|
-
@status == :wait
|
41
|
-
end
|
42
|
-
|
43
|
-
# Is the thread still alive
|
44
|
-
def alive?
|
45
|
-
@thread.alive?
|
46
|
-
end
|
47
|
-
|
48
|
-
# Current thread status
|
49
|
-
def status
|
50
|
-
@lock.synchronize{ return @status }
|
51
|
-
end
|
52
|
-
|
53
|
-
# arg:: :wait or :run
|
54
|
-
# Set current status
|
55
|
-
def status(arg)
|
56
|
-
raise InvalidType.new('Status can only be set to :wait or :run') unless arg == :wait || arg == :run
|
57
|
-
@lock.synchronize{ @status = arg }
|
58
|
-
end
|
59
|
-
|
60
|
-
# Seconds thread will wait for input
|
61
|
-
def thread_timeout
|
62
|
-
@thread_timeout
|
63
|
-
end
|
64
|
-
|
65
|
-
# Seconds thread will spend working on a given task
|
66
|
-
def action_timeout
|
67
|
-
@action_timeout
|
68
|
-
end
|
69
|
-
|
70
|
-
# t:: seconds to wait for input (floats allow for values 0 < t < 1)
|
71
|
-
# Set the maximum amount of time to wait for a task
|
72
|
-
def thread_timeout=(t)
|
73
|
-
t = t.to_f
|
74
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
75
|
-
@thread_timeout = t
|
76
|
-
t
|
77
|
-
end
|
78
|
-
|
79
|
-
# t:: seconds to work on a task (floats allow for values 0 < t < 1)
|
80
|
-
# Set the maximum amount of time to work on a given task
|
81
|
-
def action_timeout=(t)
|
82
|
-
t = t.to_f
|
83
|
-
raise ArgumentError.new('Value must be great than zero or nil') unless t > 0
|
84
|
-
@action_timeout = t
|
85
|
-
t
|
86
|
-
end
|
87
|
-
|
88
|
-
private
|
89
|
-
|
90
|
-
# Start our thread
|
91
|
-
def start_thread
|
92
|
-
begin
|
93
|
-
@logger.info("New pool thread is starting (#{self})")
|
94
|
-
until(@kill) do
|
95
|
-
status(:wait)
|
96
|
-
begin
|
97
|
-
action = nil
|
98
|
-
if(@pool.size > @pool.min)
|
99
|
-
Timeout::timeout(@thread_timeout) do
|
100
|
-
action = @pool.action
|
101
|
-
end
|
102
|
-
else
|
103
|
-
action = @pool.action
|
104
|
-
end
|
105
|
-
status(:run)
|
106
|
-
run(action[0], action[1]) unless action.nil?
|
107
|
-
status(:wait)
|
108
|
-
rescue Timeout::Error => boom
|
109
|
-
@kill = true
|
110
|
-
rescue Wakeup
|
111
|
-
@logger.info("Thread #{::Thread.current} was woken up.")
|
112
|
-
rescue Exception => boom
|
113
|
-
@logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
|
114
|
-
@respond_to.raise boom
|
115
|
-
end
|
116
|
-
end
|
117
|
-
rescue Wakeup
|
118
|
-
@logger.info("Thread #{::Thread.current} was woken up.")
|
119
|
-
rescue Exception => boom
|
120
|
-
@logger.error("Pool thread caught an exception: #{boom}\n#{boom.backtrace.join("\n")}")
|
121
|
-
@respond_to.raise boom
|
122
|
-
ensure
|
123
|
-
@logger.info("Pool thread is shutting down (#{self})")
|
124
|
-
@pool.remove(self)
|
125
|
-
@pool.create_threads
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
# action:: task to be run
|
130
|
-
# args:: arguments to be passed to task
|
131
|
-
# Run the task
|
132
|
-
def run(action, args)
|
133
|
-
begin
|
134
|
-
if(@action_timeout > 0)
|
135
|
-
Timeout::timeout(@action_timeout) do
|
136
|
-
action.call(*args.flatten)
|
137
|
-
end
|
138
|
-
else
|
139
|
-
action.call(*args[0])
|
140
|
-
end
|
141
|
-
rescue Timeout::Error => boom
|
142
|
-
@logger.warn("Pool thread reached max execution time for action: #{boom}")
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
data/test.rb
DELETED
data/tests/cases/general.rb~
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
require 'actionpool'
|
2
|
-
require 'test/unit'
|
3
|
-
|
4
|
-
class GeneralPoolTest < Test::Unit::TestCase
|
5
|
-
def setup
|
6
|
-
@pool = ActionPool::Pool.new
|
7
|
-
end
|
8
|
-
def test_numbers
|
9
|
-
assert_equal(10, @pool.size)
|
10
|
-
assert_equal(10, @pool.min)
|
11
|
-
assert_equal(100, @pool.max)
|
12
|
-
assert_equal(0, @pool.action_timeout)
|
13
|
-
assert_equal(0, @pool.thread_timeout)
|
14
|
-
assert_equal(0, @pool.action_size)
|
15
|
-
end
|
16
|
-
def test_output
|
17
|
-
a = 0
|
18
|
-
lock = Mutex.new
|
19
|
-
run = lambda{ lock.synchronize{ a += 1 } }
|
20
|
-
100.times{ @pool << run }
|
21
|
-
@pool.shutdown
|
22
|
-
assert_equal(100, a)
|
23
|
-
a = 0
|
24
|
-
jobs = [].fill(run,0,100)
|
25
|
-
@pool.add_jobs(jobs)
|
26
|
-
@pool.shutdown
|
27
|
-
assert_equal(100, a)
|
28
|
-
@pool.shutdown(true)
|
29
|
-
end
|
30
|
-
def test_args
|
31
|
-
output = nil
|
32
|
-
@pool << [lambda{|x| output = x}, [2]]
|
33
|
-
assert(2, output)
|
34
|
-
@pool.add_jobs([[lambda{|x| output = x}, [3]]])
|
35
|
-
assert(3, output)
|
36
|
-
@pool << [lambda{|x,y| output = x+y}, [1,2]]
|
37
|
-
# assert(3, output)
|
38
|
-
sleep(0.01)
|
39
|
-
output = []
|
40
|
-
@pool.add_jobs([[lambda{|x,y| output << x + y}, [1,1]], [lambda{|x| output << x}, [3]]])
|
41
|
-
# assert(output.include?(2))
|
42
|
-
# assert(output.include?(3))
|
43
|
-
@pool.shutdown(true)
|
44
|
-
end
|
45
|
-
end
|