threadlimiter 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +18 -0
- data/README +8 -4
- data/VERSION +1 -1
- data/lib/threadlimiter/enumerable.rb +10 -10
- data/lib/threadlimiter/threadlimiter.rb +60 -19
- data/lib/threadlimiter/version.rb +3 -0
- data/lib/threadlimiter.rb +1 -0
- data/test/test.rb +14 -3
- metadata +25 -12
data/CHANGELOG
CHANGED
@@ -1,3 +1,21 @@
|
|
1
|
+
0.2.0 (11-02-2012)
|
2
|
+
|
3
|
+
* Block by default.
|
4
|
+
|
5
|
+
* Added option noblock.
|
6
|
+
|
7
|
+
* Added ThreadLimiter.open.
|
8
|
+
|
9
|
+
* Added ThreadLimiter#wait.
|
10
|
+
|
11
|
+
* Removed default limit.
|
12
|
+
|
13
|
+
* Removed default number_of_clusters.
|
14
|
+
|
15
|
+
* Fixed limit=-1 when calling several methods.
|
16
|
+
|
17
|
+
* Worked around a Ruby bug in Thread.fork(*args, &block).
|
18
|
+
|
1
19
|
0.1.2 (06-10-2008)
|
2
20
|
|
3
21
|
* Introduced ThreadLimiter.handle_clusters, which is reused by
|
data/README
CHANGED
@@ -13,7 +13,7 @@ The traditional way, using Thread directly:
|
|
13
13
|
titles =
|
14
14
|
urls.collect do |url|
|
15
15
|
Thread.fork do
|
16
|
-
# ...
|
16
|
+
# ...get the title of the url...
|
17
17
|
end
|
18
18
|
end.collect do |thread|
|
19
19
|
thread.value
|
@@ -21,13 +21,15 @@ The traditional way, using Thread directly:
|
|
21
21
|
|
22
22
|
With ThreadLimiter#fork():
|
23
23
|
|
24
|
-
|
24
|
+
require "threadlimiter"
|
25
|
+
|
25
26
|
urls = [.....] # A lot of URL's. Maybe even thousends.
|
27
|
+
thread_limiter = ThreadLimiter.new(10) # Max. 10 concurrently running threads.
|
26
28
|
|
27
29
|
titles =
|
28
30
|
urls.collect do |url|
|
29
31
|
thread_limiter.fork do
|
30
|
-
# ...
|
32
|
+
# ...get the title of the url...
|
31
33
|
end
|
32
34
|
end.collect do |thread|
|
33
35
|
thread.value
|
@@ -35,9 +37,11 @@ With ThreadLimiter#fork():
|
|
35
37
|
|
36
38
|
With Enumerable#threaded_collect():
|
37
39
|
|
40
|
+
require "threadlimiter"
|
41
|
+
|
38
42
|
urls = [.....] # A lot of URL's. Maybe even thousends.
|
39
43
|
|
40
44
|
titles =
|
41
45
|
urls.threaded_collect(10) do |url| # Max. 10 concurrently running threads.
|
42
|
-
# ...
|
46
|
+
# ...get the title of the url...
|
43
47
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -24,7 +24,7 @@ module Enumerable
|
|
24
24
|
# Each cluster is run concurrently in a thread, using ThreadLimiter.new(<i>number_of_clusters</i>) and its fork().
|
25
25
|
# Set <i>number_of_clusters</i> to -1 to skip clustering.
|
26
26
|
|
27
|
-
def clustered_threaded_collect(number_of_clusters
|
27
|
+
def clustered_threaded_collect(number_of_clusters, &block)
|
28
28
|
if number_of_clusters <= 0
|
29
29
|
threaded_collect(number_of_clusters, &block)
|
30
30
|
else
|
@@ -39,7 +39,7 @@ module Enumerable
|
|
39
39
|
if limit == 0
|
40
40
|
self.select(&block)
|
41
41
|
else
|
42
|
-
self.zip(self.threaded_collect(limit
|
42
|
+
self.zip(self.threaded_collect(limit, &block)).inject([]){|r, (o, b)| r << o if b ; r}
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -50,7 +50,7 @@ module Enumerable
|
|
50
50
|
if limit == 0
|
51
51
|
self.reject(&block)
|
52
52
|
else
|
53
|
-
self.zip(self.threaded_collect(limit
|
53
|
+
self.zip(self.threaded_collect(limit, &block)).inject([]){|r, (o, b)| r << o unless b ; r}
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -61,7 +61,7 @@ module Enumerable
|
|
61
61
|
if limit == 0
|
62
62
|
self.each(&block)
|
63
63
|
else
|
64
|
-
threaded_collect(limit
|
64
|
+
threaded_collect(limit, &block)
|
65
65
|
|
66
66
|
self
|
67
67
|
end
|
@@ -71,24 +71,24 @@ module Enumerable
|
|
71
71
|
# Each cluster is run concurrently in a thread, using ThreadLimiter.new(<i>number_of_clusters</i>) and its fork().
|
72
72
|
# Set <i>number_of_clusters</i> to -1 to skip clustering.
|
73
73
|
|
74
|
-
def clustered_threaded_select(number_of_clusters
|
75
|
-
self.zip(self.clustered_threaded_collect(number_of_clusters
|
74
|
+
def clustered_threaded_select(number_of_clusters, &block)
|
75
|
+
self.zip(self.clustered_threaded_collect(number_of_clusters, &block)).inject([]){|r, (o, b)| r << o if b ; r}
|
76
76
|
end
|
77
77
|
|
78
78
|
# Like Enumerable#reject(), but all blocks are clustered.
|
79
79
|
# Each cluster is run concurrently in a thread, using ThreadLimiter.new(<i>number_of_clusters</i>) and its fork().
|
80
80
|
# Set <i>number_of_clusters</i> to -1 to skip clustering.
|
81
81
|
|
82
|
-
def clustered_threaded_reject(number_of_clusters
|
83
|
-
self.zip(self.clustered_threaded_collect(number_of_clusters
|
82
|
+
def clustered_threaded_reject(number_of_clusters, &block)
|
83
|
+
self.zip(self.clustered_threaded_collect(number_of_clusters, &block)).inject([]){|r, (o, b)| r << o unless b ; r}
|
84
84
|
end
|
85
85
|
|
86
86
|
# Like Enumerable#each(), but all blocks are clustered.
|
87
87
|
# Each cluster is run concurrently in a thread, using ThreadLimiter.new(<i>number_of_clusters</i>) and its fork().
|
88
88
|
# Set <i>number_of_clusters</i> to -1 to skip clustering.
|
89
89
|
|
90
|
-
def clustered_threaded_each(number_of_clusters
|
91
|
-
clustered_threaded_collect(number_of_clusters
|
90
|
+
def clustered_threaded_each(number_of_clusters, &block)
|
91
|
+
clustered_threaded_collect(number_of_clusters, &block)
|
92
92
|
|
93
93
|
self
|
94
94
|
end
|
@@ -4,13 +4,27 @@
|
|
4
4
|
# ThreadLimiter isn't a thread pool. Each fork really starts a new thread.
|
5
5
|
|
6
6
|
class ThreadLimiter
|
7
|
+
# Create and use a new ThreadLimiter and wait for all threads to finish.
|
8
|
+
|
9
|
+
def self.open(*args)
|
10
|
+
thread_limiter = new(*args)
|
11
|
+
|
12
|
+
begin
|
13
|
+
yield(thread_limiter)
|
14
|
+
ensure
|
15
|
+
thread_limiter.wait
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
7
19
|
# Initialize the ThreadLimiter.
|
8
20
|
# The optional parameter <i>limit</i> is the maximum number of concurrently running threads.
|
9
21
|
# Set <i>limit</i> to -1 or 0 to fork threads without limiting the number of concurrently running threads.
|
22
|
+
# Set options[:noblock] to true to start the new thread before waiting for resources.
|
10
23
|
|
11
|
-
def initialize(limit
|
24
|
+
def initialize(limit, options={})
|
12
25
|
@limit = limit # The maximum number of concurrently running threads.
|
13
26
|
@running = 0 # The number of currently running threads.
|
27
|
+
@noblock = options[:noblock]
|
14
28
|
|
15
29
|
@mutex = Mutex.new
|
16
30
|
@cv = ConditionVariable.new
|
@@ -24,36 +38,28 @@ class ThreadLimiter
|
|
24
38
|
|
25
39
|
def fork(*args, &block)
|
26
40
|
if @limit <= 0
|
27
|
-
Thread.fork
|
28
|
-
|
29
|
-
@mutex.synchronize do
|
30
|
-
while @running >= @limit
|
31
|
-
@cv.wait(@mutex)
|
32
|
-
end
|
33
|
-
|
34
|
-
@running += 1
|
41
|
+
Thread.fork do
|
42
|
+
yield(*args)
|
35
43
|
end
|
44
|
+
else
|
45
|
+
cv_wait unless @noblock
|
36
46
|
|
37
47
|
Thread.fork do
|
48
|
+
cv_wait if @noblock
|
49
|
+
|
38
50
|
begin
|
39
|
-
|
51
|
+
yield(*args)
|
40
52
|
ensure
|
41
|
-
|
42
|
-
@running -= 1
|
43
|
-
end
|
44
|
-
|
45
|
-
@cv.signal if @limit > 0
|
53
|
+
cv_signal
|
46
54
|
end
|
47
|
-
|
48
|
-
res
|
49
55
|
end
|
50
56
|
end
|
51
57
|
end
|
52
58
|
|
53
|
-
def self.handle_clusters(enumeration, number_of_clusters, method_name, &block)
|
59
|
+
def self.handle_clusters(enumeration, number_of_clusters, method_name, &block) # :nodoc:
|
54
60
|
clusters = [] # One cluster per fork.
|
55
61
|
last_pos = -1
|
56
|
-
res
|
62
|
+
res = []
|
57
63
|
|
58
64
|
enumeration.each do |object|
|
59
65
|
last_pos += 1
|
@@ -78,4 +84,39 @@ class ThreadLimiter
|
|
78
84
|
res[0..last_pos] # Remove padding nil.
|
79
85
|
end
|
80
86
|
|
87
|
+
# Wait for all threads to finish.
|
88
|
+
|
89
|
+
def wait
|
90
|
+
@mutex.synchronize do
|
91
|
+
while @running > 0
|
92
|
+
@cv.wait(@mutex)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
self
|
97
|
+
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def cv_wait
|
102
|
+
@mutex.synchronize do
|
103
|
+
while @running >= @limit
|
104
|
+
@cv.wait(@mutex)
|
105
|
+
end
|
106
|
+
|
107
|
+
@running += 1
|
108
|
+
end
|
109
|
+
|
110
|
+
self
|
111
|
+
end
|
112
|
+
|
113
|
+
def cv_signal
|
114
|
+
@mutex.synchronize do
|
115
|
+
@running -= 1
|
116
|
+
|
117
|
+
@cv.signal
|
118
|
+
end
|
119
|
+
|
120
|
+
self
|
121
|
+
end
|
81
122
|
end
|
data/lib/threadlimiter.rb
CHANGED
data/test/test.rb
CHANGED
@@ -2,9 +2,9 @@ require "test/unit"
|
|
2
2
|
require "threadlimiter"
|
3
3
|
|
4
4
|
class ThreadLimiterTest < Test::Unit::TestCase
|
5
|
-
def go(limit)
|
5
|
+
def go(limit, options={})
|
6
6
|
input = (1..100).collect{rand}
|
7
|
-
threadlimiter = ThreadLimiter.new(limit)
|
7
|
+
threadlimiter = ThreadLimiter.new(limit, options)
|
8
8
|
|
9
9
|
threads =
|
10
10
|
input.collect do |m|
|
@@ -17,7 +17,6 @@ class ThreadLimiterTest < Test::Unit::TestCase
|
|
17
17
|
|
18
18
|
assert_equal([Thread], threads.collect{|t| t.class}.uniq)
|
19
19
|
|
20
|
-
|
21
20
|
assert_equal(input.to_a , threads.collect{|t| t.value})
|
22
21
|
assert_equal(0 , threadlimiter.instance_eval{@running})
|
23
22
|
assert_equal(limit , threadlimiter.instance_eval{@limit})
|
@@ -34,6 +33,18 @@ class ThreadLimiterTest < Test::Unit::TestCase
|
|
34
33
|
def test_with_zero_limit
|
35
34
|
go(0)
|
36
35
|
end
|
36
|
+
|
37
|
+
def test_with_limit_with_noblock
|
38
|
+
go(10, :noblock=>true)
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_with_no_limit_with_noblock
|
42
|
+
go(-1, :noblock=>true)
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_with_zero_limit_with_noblock
|
46
|
+
go(0, :noblock=>true)
|
47
|
+
end
|
37
48
|
end
|
38
49
|
|
39
50
|
class ThreadLimiterEnumerableTest < Test::Unit::TestCase
|
metadata
CHANGED
@@ -1,7 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: threadlimiter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
4
|
+
hash: 23
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 0
|
10
|
+
version: 0.2.0
|
5
11
|
platform: ruby
|
6
12
|
authors:
|
7
13
|
- Erik Veenstra
|
@@ -9,8 +15,7 @@ autorequire:
|
|
9
15
|
bindir: bin
|
10
16
|
cert_chain: []
|
11
17
|
|
12
|
-
date:
|
13
|
-
default_executable:
|
18
|
+
date: 2012-02-11 00:00:00 Z
|
14
19
|
dependencies: []
|
15
20
|
|
16
21
|
description: Fork threads like Thread.fork, but limit the number of concurrently running threads.
|
@@ -22,16 +27,18 @@ extensions: []
|
|
22
27
|
extra_rdoc_files: []
|
23
28
|
|
24
29
|
files:
|
25
|
-
- lib/threadlimiter
|
26
|
-
- lib/threadlimiter/threadlimiter.rb
|
27
|
-
- lib/threadlimiter/enumerable.rb
|
28
30
|
- lib/threadlimiter.rb
|
31
|
+
- lib/threadlimiter/enumerable.rb
|
32
|
+
- lib/threadlimiter/threadlimiter.rb
|
33
|
+
- lib/threadlimiter/version.rb
|
29
34
|
- README
|
30
35
|
- LICENSE
|
31
36
|
- VERSION
|
32
37
|
- CHANGELOG
|
33
|
-
|
38
|
+
- test/test.rb
|
34
39
|
homepage: http://www.erikveen.dds.nl/threadlimiter/index.html
|
40
|
+
licenses: []
|
41
|
+
|
35
42
|
post_install_message:
|
36
43
|
rdoc_options:
|
37
44
|
- README
|
@@ -39,29 +46,35 @@ rdoc_options:
|
|
39
46
|
- VERSION
|
40
47
|
- CHANGELOG
|
41
48
|
- --title
|
42
|
-
- threadlimiter (0.
|
49
|
+
- threadlimiter (0.2.0)
|
43
50
|
- --main
|
44
51
|
- README
|
45
52
|
require_paths:
|
46
53
|
- lib
|
47
54
|
required_ruby_version: !ruby/object:Gem::Requirement
|
55
|
+
none: false
|
48
56
|
requirements:
|
49
57
|
- - ">="
|
50
58
|
- !ruby/object:Gem::Version
|
59
|
+
hash: 3
|
60
|
+
segments:
|
61
|
+
- 0
|
51
62
|
version: "0"
|
52
|
-
version:
|
53
63
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
64
|
+
none: false
|
54
65
|
requirements:
|
55
66
|
- - ">="
|
56
67
|
- !ruby/object:Gem::Version
|
68
|
+
hash: 3
|
69
|
+
segments:
|
70
|
+
- 0
|
57
71
|
version: "0"
|
58
|
-
version:
|
59
72
|
requirements: []
|
60
73
|
|
61
74
|
rubyforge_project: threadlimiter
|
62
|
-
rubygems_version: 1.
|
75
|
+
rubygems_version: 1.8.12
|
63
76
|
signing_key:
|
64
|
-
specification_version:
|
77
|
+
specification_version: 3
|
65
78
|
summary: Fork threads like Thread.fork, but limit the number of concurrently running threads.
|
66
79
|
test_files:
|
67
80
|
- test/test.rb
|