forkandreturn 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +17 -0
- data/README +5 -1
- data/VERSION +1 -1
- data/lib/forkandreturn/enumerable.rb +6 -8
- data/lib/forkandreturn/forkandreturn.rb +61 -4
- data/test/test.rb +10 -0
- metadata +5 -4
data/CHANGELOG
CHANGED
@@ -1,3 +1,20 @@
|
|
1
|
+
0.2.0 (27-07-2008)
|
2
|
+
|
3
|
+
* Removed Enumerable#concurrent() and
|
4
|
+
Enumerable#clustered_concurrent(). (Use
|
5
|
+
Enumerable#concurrent_collect() and
|
6
|
+
Enumerable#clustered_concurrent_collect() instead,
|
7
|
+
respectively.)
|
8
|
+
|
9
|
+
* Fixed the clustering of empty enumerables.
|
10
|
+
|
11
|
+
* Added File.owned?(), so the temporary file with the
|
12
|
+
intermediate results can't be replaced by other people.
|
13
|
+
|
14
|
+
* Reduced the overhead of the clustering.
|
15
|
+
|
16
|
+
* Reduced the overhead of Marshal.
|
17
|
+
|
1
18
|
0.1.1 (19-07-2008)
|
2
19
|
|
3
20
|
* Added example.txt.
|
data/README
CHANGED
@@ -1,8 +1,12 @@
|
|
1
|
-
ForkAndReturn implements a couple of methods that
|
1
|
+
ForkAndReturn implements a couple of methods that simplify
|
2
2
|
running a block of code in a subprocess. The result (Ruby
|
3
3
|
object or exception) of the block will be available in the
|
4
4
|
parent process.
|
5
5
|
|
6
|
+
ForkAndReturn also enriches Enumerable with a couple of methods
|
7
|
+
(e.g. Enumerable#concurrent_collect()), in order to simplify
|
8
|
+
the concurrent execution of a block for a collection of objects.
|
9
|
+
|
6
10
|
The intermediate return value (or exception) will be
|
7
11
|
Marshal'led to disk. This means that it is possible to
|
8
12
|
(concurrently) run thousands of child process, with a relative
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -74,14 +74,14 @@ module Enumerable
|
|
74
74
|
if number_of_clusters < 1
|
75
75
|
self.concurrent_collect(number_of_clusters, &block)
|
76
76
|
else
|
77
|
-
clusters = []
|
78
|
-
last_pos =
|
77
|
+
clusters = [] # One cluster per fork.
|
78
|
+
last_pos = -1
|
79
79
|
res = []
|
80
80
|
|
81
|
-
self.
|
82
|
-
|
81
|
+
self.each do |object|
|
82
|
+
last_pos += 1
|
83
83
|
|
84
|
-
last_pos
|
84
|
+
(clusters[last_pos%number_of_clusters] ||= []) << object
|
85
85
|
end
|
86
86
|
|
87
87
|
clusters.concurrent_collect(-1) do |cluster|
|
@@ -96,7 +96,7 @@ module Enumerable
|
|
96
96
|
res.concat(array)
|
97
97
|
end
|
98
98
|
|
99
|
-
res[0..last_pos] # Remove padding
|
99
|
+
res[0..last_pos] # Remove padding nil.
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
@@ -140,8 +140,6 @@ module Enumerable
|
|
140
140
|
self
|
141
141
|
end
|
142
142
|
|
143
|
-
alias concurrent concurrent_collect
|
144
143
|
alias concurrent_map concurrent_collect
|
145
|
-
alias clustered_concurrent clustered_concurrent_collect
|
146
144
|
alias clustered_concurrent_map clustered_concurrent_collect
|
147
145
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# ForkAndReturn implements a couple of methods that
|
1
|
+
# ForkAndReturn implements a couple of methods that simplify running a block of code in a subprocess.
|
2
2
|
# The result (Ruby object or exception) of the block will be available in the parent process.
|
3
3
|
#
|
4
4
|
# The intermediate return value (or exception) will be Marshal'led to disk.
|
@@ -27,14 +27,14 @@ module ForkAndReturn
|
|
27
27
|
# [1, 2, 3, 4].collect do |object|
|
28
28
|
# Thread.fork do
|
29
29
|
# ForkAndReturn.fork_and_return do
|
30
|
-
# 2
|
30
|
+
# object*2
|
31
31
|
# end
|
32
32
|
# end
|
33
33
|
# end.collect do |thread|
|
34
34
|
# thread.value
|
35
35
|
# end # ===> [2, 4, 6, 8]
|
36
36
|
#
|
37
|
-
# This runs each "2
|
37
|
+
# This runs each "object*2" statement in a seperate process, concurrently.
|
38
38
|
# Hopefully, the processes are spread over all available CPU's.
|
39
39
|
# That's a simple way of parallel processing!
|
40
40
|
# (Although Enumerable#concurrent_collect() is even simpler...)
|
@@ -53,6 +53,18 @@ module ForkAndReturn
|
|
53
53
|
# If you call it, the WAITing, LOADing and RESULTing (explained in fork_and_return_core()) will be performed in one go.
|
54
54
|
#
|
55
55
|
# <i>*args</i> is passed to the block.
|
56
|
+
#
|
57
|
+
# Example:
|
58
|
+
#
|
59
|
+
# [1, 2, 3, 4].collect do |object|
|
60
|
+
# ForkAndReturn.fork_and_return_later do
|
61
|
+
# object*2
|
62
|
+
# end
|
63
|
+
# end.collect do |wait|
|
64
|
+
# wait.call
|
65
|
+
# end # ===> [2, 4, 6, 8]
|
66
|
+
#
|
67
|
+
# This runs each "object*2" statement in a seperate process, concurrently.
|
56
68
|
|
57
69
|
def self.fork_and_return_later(*args, &block)
|
58
70
|
wait = fork_and_return_core(*args, &block)
|
@@ -72,10 +84,51 @@ module ForkAndReturn
|
|
72
84
|
# If you call RESULT-lambda, the result of the child process will be handled.
|
73
85
|
# This means either "return the return value of the block" or "raise the exception"
|
74
86
|
#
|
87
|
+
# fork_and_return_core() is coded like this:
|
88
|
+
#
|
89
|
+
# def fork_and_return_core
|
90
|
+
# # Fork a process.
|
91
|
+
#
|
92
|
+
# lambda do
|
93
|
+
# # Wait for the result.
|
94
|
+
#
|
95
|
+
# lambda do
|
96
|
+
# # Load the result and delete the temp file.
|
97
|
+
#
|
98
|
+
# lambda do
|
99
|
+
# # Handle the result.
|
100
|
+
# end
|
101
|
+
# end
|
102
|
+
# end
|
103
|
+
# end
|
104
|
+
#
|
105
|
+
# fork_and_return_core() is used like this:
|
106
|
+
#
|
107
|
+
# wait = ForkAndReturn.fork_and_return_core{raise "BOOM"}
|
108
|
+
# load = wait.call
|
109
|
+
# result = load.call
|
110
|
+
# value = result.call # This is were the exception "BOOM" is raised.
|
111
|
+
#
|
75
112
|
# at_exit blocks defined in the child itself will be executed in the child,
|
76
113
|
# whereas at_exit blocks defined in the parent won't be executed in the child.
|
77
114
|
#
|
78
115
|
# <i>*args</i> is passed to the block.
|
116
|
+
#
|
117
|
+
# Example:
|
118
|
+
#
|
119
|
+
# [1, 2, 3, 4].collect do |object|
|
120
|
+
# ForkAndReturn.fork_and_return do
|
121
|
+
# object*2
|
122
|
+
# end
|
123
|
+
# end.collect do |wait|
|
124
|
+
# wait.call
|
125
|
+
# end.collect do |load|
|
126
|
+
# load.call
|
127
|
+
# end.collect do |result|
|
128
|
+
# result.call
|
129
|
+
# end # ===> [2, 4, 6, 8]
|
130
|
+
#
|
131
|
+
# This runs each "object*2" statement in a seperate process, concurrently.
|
79
132
|
|
80
133
|
def self.fork_and_return_core(*args, &block)
|
81
134
|
file = Util.tempfile
|
@@ -113,7 +166,11 @@ module ForkAndReturn
|
|
113
166
|
|
114
167
|
lambda do # Load the result and delete the temp file.
|
115
168
|
begin
|
116
|
-
|
169
|
+
if File.owned?(file)
|
170
|
+
ok, res = File.open(file, "rb"){|f| Marshal.load(f.read)}
|
171
|
+
else
|
172
|
+
ok, res = false, WorkerError.new("you're not the owner of the temporary file")
|
173
|
+
end
|
117
174
|
rescue Errno::ENOENT # No such file or directory
|
118
175
|
ok, res = false, WorkerError.new("the worker hasn't returned a result")
|
119
176
|
rescue EOFError # end of file reached
|
data/test/test.rb
CHANGED
@@ -92,6 +92,16 @@ class ForkAndReturnEnumerableTest < Test::Unit::TestCase
|
|
92
92
|
class ForkAndReturnEnumerableTestException < StandardError
|
93
93
|
end
|
94
94
|
|
95
|
+
def test_empty_array
|
96
|
+
assert_equal([], [].concurrent_collect(0){2})
|
97
|
+
assert_equal([], [].concurrent_collect(3){2})
|
98
|
+
assert_equal([], [].concurrent_collect(-1){2})
|
99
|
+
|
100
|
+
assert_equal([], [].clustered_concurrent_collect(0){2})
|
101
|
+
assert_equal([], [].clustered_concurrent_collect(3){2})
|
102
|
+
assert_equal([], [].clustered_concurrent_collect(-1){2})
|
103
|
+
end
|
104
|
+
|
95
105
|
def test_array
|
96
106
|
data = (1..10).to_a
|
97
107
|
block = lambda{|n| n**n}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forkandreturn
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Erik Veenstra
|
@@ -9,11 +9,12 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-07-
|
12
|
+
date: 2008-07-27 00:00:00 +02:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: threadlimiter
|
17
|
+
type: :runtime
|
17
18
|
version_requirement:
|
18
19
|
version_requirements: !ruby/object:Gem::Requirement
|
19
20
|
requirements:
|
@@ -51,7 +52,7 @@ rdoc_options:
|
|
51
52
|
- CHANGELOG
|
52
53
|
- example.txt
|
53
54
|
- --title
|
54
|
-
- forkandreturn (0.
|
55
|
+
- forkandreturn (0.2.0)
|
55
56
|
- --main
|
56
57
|
- README
|
57
58
|
require_paths:
|
@@ -71,7 +72,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
72
|
requirements: []
|
72
73
|
|
73
74
|
rubyforge_project: forkandreturn
|
74
|
-
rubygems_version: 1.
|
75
|
+
rubygems_version: 1.2.0
|
75
76
|
signing_key:
|
76
77
|
specification_version: 2
|
77
78
|
summary: Runs a block of code in a seperate process and collects the result later. Includes a lot of convenient methods on Enumerable.
|