threadz 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +13 -3
- data/VERSION +1 -1
- data/lib/threadz/atomic_integer.rb +12 -0
- data/lib/threadz/batch.rb +9 -17
- data/lib/threadz/thread_pool.rb +2 -1
- data/spec/basic/thread_pool_spec.rb +41 -0
- data/spec/performance/batch_spec.rb +29 -0
- data/spec/threadz_spec.rb +19 -74
- data/threadz.gemspec +9 -4
- metadata +8 -4
data/Rakefile
CHANGED
@@ -32,7 +32,17 @@ end
|
|
32
32
|
|
33
33
|
desc "Run all test cases"
|
34
34
|
task 'spec' do |task|
|
35
|
-
exec 'spec -c spec/*.rb'
|
35
|
+
exec 'spec -c -f n spec/*.rb spec/basic/*.rb'
|
36
|
+
end
|
37
|
+
|
38
|
+
desc "Run all performance-oriented test cases"
|
39
|
+
task 'spec:performance' do |task|
|
40
|
+
exec 'spec -c -f n -t 30.0 spec/spec_helper.rb spec/performance/*.rb'
|
41
|
+
end
|
42
|
+
|
43
|
+
desc "Run *all* specs"
|
44
|
+
task 'spec:all' do |task|
|
45
|
+
exec 'spec -c -f n -t 30.0 spec/spec_helper.rb spec/**/*.rb'
|
36
46
|
end
|
37
47
|
|
38
48
|
desc "Run all test cases 10 times (or n times)"
|
@@ -114,8 +124,8 @@ begin
|
|
114
124
|
require 'jeweler'
|
115
125
|
Jeweler::Tasks.new do |gemspec|
|
116
126
|
gemspec.name = "threadz"
|
117
|
-
gemspec.summary = "
|
118
|
-
|
127
|
+
gemspec.summary = "An easy Ruby threadpool library."
|
128
|
+
gemspec.description = "A Ruby threadpool library to handle threadpools and make batch jobs easier."
|
119
129
|
gemspec.email = "nanodeath@gmail.com"
|
120
130
|
gemspec.homepage = "http://github.com/nanodeath/threadz"
|
121
131
|
gemspec.authors = ["Max Aller"]
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.3
|
@@ -1,6 +1,10 @@
|
|
1
1
|
require 'thread'
|
2
2
|
|
3
3
|
module Threadz
|
4
|
+
# Related to, or at least named after, Java's AtomicInteger.
|
5
|
+
# Provides a thread-safe integer counter thing.
|
6
|
+
# The code used in this file, while slightly verbose, is to optimize
|
7
|
+
# performance. Avoiding additional method calls and blocks is preferred.
|
4
8
|
class AtomicInteger
|
5
9
|
def initialize(value)
|
6
10
|
@value = value
|
@@ -20,9 +24,17 @@ module Threadz
|
|
20
24
|
end
|
21
25
|
|
22
26
|
def decrement(amount=1)
|
27
|
+
# We could refactor and just call set here, but it's faster just to write
|
28
|
+
# the extra two lines.
|
23
29
|
@mutex.lock
|
24
30
|
@value -= amount
|
25
31
|
@mutex.unlock
|
26
32
|
end
|
33
|
+
|
34
|
+
def set(value)
|
35
|
+
@mutex.lock
|
36
|
+
@value = value
|
37
|
+
@mutex.unlock
|
38
|
+
end
|
27
39
|
end
|
28
40
|
end
|
data/lib/threadz/batch.rb
CHANGED
@@ -55,24 +55,20 @@ module Threadz
|
|
55
55
|
# +:timeout+:: If specified, will only wait for at least this many seconds
|
56
56
|
# for the batch to finish. Typically used with #completed?
|
57
57
|
def wait_until_done(opts={})
|
58
|
-
return if completed?
|
59
|
-
|
60
58
|
raise "Threadz: thread deadlocked because batch job was never started" if @latent && !@started
|
61
59
|
|
62
60
|
timeout = opts.key?(:timeout) ? opts[:timeout] : 0
|
63
|
-
|
64
|
-
|
65
|
-
@sleeper.wait(timeout)
|
61
|
+
@sleeper.wait(timeout) unless completed?
|
66
62
|
end
|
67
63
|
|
68
|
-
# Returns true iff there are no
|
64
|
+
# Returns true iff there are no jobs outstanding.
|
69
65
|
def completed?
|
70
66
|
return @jobs_count.value == 0
|
71
67
|
end
|
72
68
|
|
73
69
|
# If this is a latent batch, start processing all of the jobs in the queue.
|
74
70
|
def start
|
75
|
-
|
71
|
+
@job_lock.synchronize { # in case another thread tries to push new jobs onto the queue while we're starting
|
76
72
|
if @latent
|
77
73
|
@started = true
|
78
74
|
until @job_queue.empty?
|
@@ -82,19 +78,13 @@ module Threadz
|
|
82
78
|
else
|
83
79
|
return false
|
84
80
|
end
|
85
|
-
|
81
|
+
}
|
86
82
|
end
|
87
83
|
|
88
84
|
# Execute a given block when the batch has finished processing. If the batch
|
89
85
|
# has already finished executing, execute immediately.
|
90
86
|
def when_done(&block)
|
91
|
-
@job_lock.
|
92
|
-
if completed?
|
93
|
-
block.call
|
94
|
-
else
|
95
|
-
@when_done_blocks << block
|
96
|
-
end
|
97
|
-
@job_lock.unlock
|
87
|
+
@job_lock.synchronize { completed? ? block.call : @when_done_blocks << block }
|
98
88
|
end
|
99
89
|
|
100
90
|
private
|
@@ -110,11 +100,13 @@ module Threadz
|
|
110
100
|
@threadpool.process do
|
111
101
|
job.call
|
112
102
|
# Lock in case we get two threads at the "fork in the road" at the same time
|
113
|
-
|
103
|
+
# Note: locking here actually creates undesirable behavior. Still investigating why,
|
104
|
+
# seems like it should be useful.
|
105
|
+
#@job_lock.lock
|
114
106
|
@jobs_count.decrement
|
115
107
|
# fork in the road
|
116
108
|
handle_done if completed?
|
117
|
-
|
109
|
+
#@job_lock.unlock
|
118
110
|
end
|
119
111
|
end
|
120
112
|
end
|
data/lib/threadz/thread_pool.rb
CHANGED
@@ -80,7 +80,8 @@ module Threadz
|
|
80
80
|
|
81
81
|
# Kill a thread after it completes its current job
|
82
82
|
def kill_thread
|
83
|
-
|
83
|
+
# TODO: ideally this would be unshift, but Queues don't have that. Come up with an alternative.
|
84
|
+
@queue << Directive::SUICIDE_PILL
|
84
85
|
end
|
85
86
|
|
86
87
|
# This thread watches over the pool and allocated and deallocates threads
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
describe Threadz::ThreadPool do
|
4
|
+
before(:each) do
|
5
|
+
@T = Threadz::ThreadPool.new
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should perform well for IO jobs" do
|
9
|
+
urls = []
|
10
|
+
urls << "http://www.google.com/" << "http://www.yahoo.com/" << 'http://www.microsoft.com/'
|
11
|
+
urls << "http://www.cnn.com/" << "http://slashdot.org/" << "http://www.mozilla.org/"
|
12
|
+
urls << "http://www.ubuntu.com/" << "http://github.com/"
|
13
|
+
time_single_threaded = Time.now
|
14
|
+
|
15
|
+
begin
|
16
|
+
(urls * 2).each do |url|
|
17
|
+
response = Net::HTTP.get_response(URI.parse(url))
|
18
|
+
body = response.body
|
19
|
+
end
|
20
|
+
|
21
|
+
time_single_threaded = Time.now - time_single_threaded
|
22
|
+
|
23
|
+
time_multi_threaded = Time.now
|
24
|
+
b = @T.new_batch
|
25
|
+
(urls * 2).each do |url|
|
26
|
+
b << Proc.new do
|
27
|
+
response = Net::HTTP.get_response(URI.parse(url))
|
28
|
+
body = response.body
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
b.wait_until_done
|
33
|
+
time_multi_threaded = Time.now - time_multi_threaded
|
34
|
+
|
35
|
+
time_multi_threaded.should < time_single_threaded
|
36
|
+
|
37
|
+
rescue SocketError
|
38
|
+
pending "pending working internet connection"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
3
|
+
describe Threadz::Batch do
|
4
|
+
before(:each) do
|
5
|
+
@T = Threadz::ThreadPool.new(:initial_size => 25)
|
6
|
+
end
|
7
|
+
|
8
|
+
it "shouldn't fail under load" do
|
9
|
+
jobs = 1000
|
10
|
+
times_per_job = 1000
|
11
|
+
i = ::Threadz::AtomicInteger.new(0)
|
12
|
+
|
13
|
+
b1 = @T.new_batch(:latent => true)
|
14
|
+
b2 = @T.new_batch(:latent => true)
|
15
|
+
|
16
|
+
jobs.times do
|
17
|
+
b1 << lambda { times_per_job.times { i.increment } }
|
18
|
+
b2 << lambda { times_per_job.times { i.decrement } }
|
19
|
+
end
|
20
|
+
|
21
|
+
b1.start
|
22
|
+
b2.start
|
23
|
+
|
24
|
+
b1.wait_until_done
|
25
|
+
b2.wait_until_done
|
26
|
+
|
27
|
+
i.value.should == 0
|
28
|
+
end
|
29
|
+
end
|
data/spec/threadz_spec.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
|
2
2
|
require 'spec_helper'
|
3
3
|
|
4
|
-
require 'net/http'
|
5
|
-
|
6
4
|
describe Threadz do
|
7
5
|
describe Threadz::ThreadPool do
|
8
6
|
before(:each) do
|
@@ -26,40 +24,6 @@ describe Threadz do
|
|
26
24
|
lambda { @T.new_batch(:latent => true) }.should_not raise_error
|
27
25
|
end
|
28
26
|
|
29
|
-
it "should perform well for IO jobs" do
|
30
|
-
urls = []
|
31
|
-
urls << "http://www.google.com/" << "http://www.yahoo.com/" << 'http://www.microsoft.com/'
|
32
|
-
urls << "http://www.cnn.com/" << "http://slashdot.org/" << "http://www.mozilla.org/"
|
33
|
-
urls << "http://www.ubuntu.com/" << "http://github.com/"
|
34
|
-
time_single_threaded = Time.now
|
35
|
-
|
36
|
-
begin
|
37
|
-
(urls * 3).each do |url|
|
38
|
-
response = Net::HTTP.get_response(URI.parse(url))
|
39
|
-
body = response.body
|
40
|
-
end
|
41
|
-
|
42
|
-
time_single_threaded = Time.now - time_single_threaded
|
43
|
-
|
44
|
-
time_multi_threaded = Time.now
|
45
|
-
b = @T.new_batch
|
46
|
-
(urls * 3).each do |url|
|
47
|
-
b << Proc.new do
|
48
|
-
response = Net::HTTP.get_response(URI.parse(url))
|
49
|
-
body = response.body
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
b.wait_until_done
|
54
|
-
time_multi_threaded = Time.now - time_multi_threaded
|
55
|
-
|
56
|
-
time_multi_threaded.should < time_single_threaded
|
57
|
-
|
58
|
-
rescue SocketError
|
59
|
-
pending "pending working internet connection"
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
27
|
describe Threadz::Batch do
|
64
28
|
it "should support jobs" do
|
65
29
|
i = 0
|
@@ -108,7 +72,7 @@ describe Threadz do
|
|
108
72
|
@i.should == 3
|
109
73
|
end
|
110
74
|
|
111
|
-
it "should support latent option
|
75
|
+
it "should support latent option" do
|
112
76
|
i = 0
|
113
77
|
b = @T.new_batch(:latent => true)
|
114
78
|
b << lambda { i += 1 }
|
@@ -172,11 +136,11 @@ describe Threadz do
|
|
172
136
|
end
|
173
137
|
|
174
138
|
it "should support 'when_done'" do
|
175
|
-
i = 0
|
139
|
+
i = ::Threadz::AtomicInteger.new(0)
|
176
140
|
when_done_executed = false
|
177
141
|
b = @T.new_batch(:latent => true)
|
178
142
|
|
179
|
-
100.times { b << lambda { i
|
143
|
+
100.times { b << lambda { i.increment } }
|
180
144
|
|
181
145
|
b.when_done { when_done_executed = true }
|
182
146
|
|
@@ -184,8 +148,12 @@ describe Threadz do
|
|
184
148
|
|
185
149
|
b.start
|
186
150
|
|
187
|
-
|
151
|
+
Timeout::timeout(10) do
|
152
|
+
sleep(0.1) until i.value == 100
|
153
|
+
end
|
188
154
|
|
155
|
+
i.value.should == 100
|
156
|
+
|
189
157
|
b.completed?.should be_true
|
190
158
|
when_done_executed.should be_true
|
191
159
|
end
|
@@ -193,11 +161,11 @@ describe Threadz do
|
|
193
161
|
it "should call 'when_done' immediately when batch is already done" do
|
194
162
|
i = 0
|
195
163
|
when_done_executed = false
|
196
|
-
b = @T.new_batch
|
164
|
+
b = @T.new_batch :latent => true
|
197
165
|
|
198
|
-
|
199
|
-
|
200
|
-
|
166
|
+
100.times { b << lambda { i += 1 } }
|
167
|
+
|
168
|
+
b.start
|
201
169
|
|
202
170
|
b.wait_until_done
|
203
171
|
|
@@ -211,42 +179,19 @@ describe Threadz do
|
|
211
179
|
it "should support multiple 'when_done' blocks" do
|
212
180
|
i = 0
|
213
181
|
when_done_executed = 0
|
214
|
-
b = @T.new_batch
|
182
|
+
b = @T.new_batch :latent => true
|
215
183
|
|
216
|
-
|
217
|
-
# the batch is already finished, so wrapping in Thread#exclusive
|
218
|
-
Thread.exclusive do
|
219
|
-
100.times { b << lambda { i += 1 } }
|
220
|
-
end
|
184
|
+
100.times { b << lambda { i += 1 } }
|
221
185
|
|
222
|
-
|
186
|
+
10.times { b.when_done { when_done_executed += 1 } }
|
223
187
|
|
224
|
-
sleep(0.1)
|
225
188
|
|
226
|
-
|
227
|
-
when_done_executed.should == 3
|
228
|
-
end
|
189
|
+
b.start
|
229
190
|
|
230
|
-
|
231
|
-
jobs = 1000
|
232
|
-
times_per_job = 100
|
233
|
-
i = ::Threadz::AtomicInteger.new(0)
|
234
|
-
|
235
|
-
b1 = @T.new_batch(:latent => true)
|
236
|
-
b2 = @T.new_batch(:latent => true)
|
237
|
-
|
238
|
-
jobs.times do
|
239
|
-
b1 << lambda { times_per_job.times { i.increment } }
|
240
|
-
b2 << lambda { times_per_job.times { i.decrement } }
|
241
|
-
end
|
242
|
-
|
243
|
-
b1.start
|
244
|
-
b2.start
|
245
|
-
|
246
|
-
b1.wait_until_done
|
247
|
-
b2.wait_until_done
|
191
|
+
b.wait_until_done
|
248
192
|
|
249
|
-
|
193
|
+
b.completed?.should be_true
|
194
|
+
when_done_executed.should == 10
|
250
195
|
end
|
251
196
|
end
|
252
197
|
end
|
data/threadz.gemspec
CHANGED
@@ -5,11 +5,12 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{threadz}
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.3"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Max Aller"]
|
12
|
-
s.date = %q{
|
12
|
+
s.date = %q{2010-02-01}
|
13
|
+
s.description = %q{A Ruby threadpool library to handle threadpools and make batch jobs easier.}
|
13
14
|
s.email = %q{nanodeath@gmail.com}
|
14
15
|
s.extra_rdoc_files = [
|
15
16
|
"README.rdoc"
|
@@ -28,6 +29,8 @@ Gem::Specification.new do |s|
|
|
28
29
|
"lib/threadz/sleeper.rb",
|
29
30
|
"lib/threadz/thread_pool.rb",
|
30
31
|
"spec/atomic_integer_spec.rb",
|
32
|
+
"spec/basic/thread_pool_spec.rb",
|
33
|
+
"spec/performance/batch_spec.rb",
|
31
34
|
"spec/spec_helper.rb",
|
32
35
|
"spec/threadz_spec.rb",
|
33
36
|
"threadz.gemspec"
|
@@ -36,11 +39,13 @@ Gem::Specification.new do |s|
|
|
36
39
|
s.rdoc_options = ["--charset=UTF-8"]
|
37
40
|
s.require_paths = ["lib"]
|
38
41
|
s.rubygems_version = %q{1.3.5}
|
39
|
-
s.summary = %q{
|
42
|
+
s.summary = %q{An easy Ruby threadpool library.}
|
40
43
|
s.test_files = [
|
41
44
|
"spec/atomic_integer_spec.rb",
|
42
45
|
"spec/threadz_spec.rb",
|
43
|
-
"spec/
|
46
|
+
"spec/performance/batch_spec.rb",
|
47
|
+
"spec/spec_helper.rb",
|
48
|
+
"spec/basic/thread_pool_spec.rb"
|
44
49
|
]
|
45
50
|
|
46
51
|
if s.respond_to? :specification_version then
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: threadz
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Max Aller
|
@@ -9,11 +9,11 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2010-02-01 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
16
|
-
description:
|
16
|
+
description: A Ruby threadpool library to handle threadpools and make batch jobs easier.
|
17
17
|
email: nanodeath@gmail.com
|
18
18
|
executables: []
|
19
19
|
|
@@ -35,6 +35,8 @@ files:
|
|
35
35
|
- lib/threadz/sleeper.rb
|
36
36
|
- lib/threadz/thread_pool.rb
|
37
37
|
- spec/atomic_integer_spec.rb
|
38
|
+
- spec/basic/thread_pool_spec.rb
|
39
|
+
- spec/performance/batch_spec.rb
|
38
40
|
- spec/spec_helper.rb
|
39
41
|
- spec/threadz_spec.rb
|
40
42
|
- threadz.gemspec
|
@@ -65,8 +67,10 @@ rubyforge_project:
|
|
65
67
|
rubygems_version: 1.3.5
|
66
68
|
signing_key:
|
67
69
|
specification_version: 3
|
68
|
-
summary:
|
70
|
+
summary: An easy Ruby threadpool library.
|
69
71
|
test_files:
|
70
72
|
- spec/atomic_integer_spec.rb
|
71
73
|
- spec/threadz_spec.rb
|
74
|
+
- spec/performance/batch_spec.rb
|
72
75
|
- spec/spec_helper.rb
|
76
|
+
- spec/basic/thread_pool_spec.rb
|