forkoff 0.0.1 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README +103 -44
- data/gemspec.rb +1 -1
- data/install.rb +0 -0
- data/lib/forkoff.rb +136 -28
- data/samples/a.rb +0 -1
- data/samples/b.rb +0 -1
- data/samples/c.rb +0 -1
- data/samples/d.rb +10 -0
- metadata +6 -6
- data/HISTORY +0 -34
data/README
CHANGED
@@ -20,13 +20,52 @@ DESCRIPTION
|
|
20
20
|
child process and collecting the results. forkoff can limit the number of
|
21
21
|
child processes which is, by default, 2.
|
22
22
|
|
23
|
+
HISTORY
|
24
|
+
0.0.4
|
25
|
+
- code re-org
|
26
|
+
- add :strategy option
|
27
|
+
- default number of processes is 2, not 8
|
28
|
+
|
29
|
+
0.0.1
|
30
|
+
|
31
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
32
|
+
channel. in this way the producers do not build up a massize parllel data
|
33
|
+
structure but provide data to the consumers only as fast as they can fork
|
34
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
35
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
36
|
+
thread popping of jobs, forking, and yielding results.
|
37
|
+
|
38
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
39
|
+
of arrays which removed some sync overhead.
|
40
|
+
|
41
|
+
- you can configure the number of processes globally with
|
42
|
+
|
43
|
+
Forkoff.default['proccess'] = 4
|
44
|
+
|
45
|
+
- you can now pass either an options hash
|
46
|
+
|
47
|
+
forkoff( :processes => 2 ) ...
|
48
|
+
|
49
|
+
or plain vanilla number
|
50
|
+
|
51
|
+
forkoff( 2 ) ...
|
52
|
+
|
53
|
+
to the forkoff call
|
54
|
+
|
55
|
+
- default number of processes is 8, not 2
|
56
|
+
|
57
|
+
|
58
|
+
0.0.0
|
59
|
+
|
60
|
+
initial version
|
61
|
+
|
62
|
+
|
23
63
|
SAMPLES
|
24
64
|
|
25
65
|
<========< samples/a.rb >========>
|
26
66
|
|
27
67
|
~ > cat samples/a.rb
|
28
68
|
|
29
|
-
#
|
30
69
|
# forkoff makes it trivial to do parallel processing with ruby, the following
|
31
70
|
# prints out each word in a separate process
|
32
71
|
#
|
@@ -37,15 +76,14 @@ SAMPLES
|
|
37
76
|
|
38
77
|
~ > ruby samples/a.rb
|
39
78
|
|
40
|
-
hey from
|
41
|
-
you from
|
79
|
+
hey from 1032
|
80
|
+
you from 1033
|
42
81
|
|
43
82
|
|
44
83
|
<========< samples/b.rb >========>
|
45
84
|
|
46
85
|
~ > cat samples/b.rb
|
47
86
|
|
48
|
-
#
|
49
87
|
# for example, this takes only 4 seconds or so to complete (8 iterations
|
50
88
|
# running in two processes = twice as fast)
|
51
89
|
#
|
@@ -69,7 +107,7 @@ SAMPLES
|
|
69
107
|
|
70
108
|
~ > ruby samples/b.rb
|
71
109
|
|
72
|
-
elapsed: 4.
|
110
|
+
elapsed: 4.25545883178711
|
73
111
|
results: [0, 1, 4, 9, 16, 25, 36, 49]
|
74
112
|
|
75
113
|
|
@@ -77,7 +115,6 @@ SAMPLES
|
|
77
115
|
|
78
116
|
~ > cat samples/c.rb
|
79
117
|
|
80
|
-
#
|
81
118
|
# forkoff does *NOT* spawn processes in batches, waiting for each batch to
|
82
119
|
# complete. rather, it keeps a certain number of processes busy until all
|
83
120
|
# results have been gathered. in otherwords the following will ensure that 3
|
@@ -114,70 +151,92 @@ SAMPLES
|
|
114
151
|
~ > ruby samples/c.rb
|
115
152
|
|
116
153
|
|
117
|
-
pid:
|
118
|
-
elapsed: 3.
|
154
|
+
pid: 1048
|
155
|
+
elapsed: 3.14415812492371
|
119
156
|
|
120
157
|
---
|
121
158
|
a: |
|
122
|
-
-+-
|
123
|
-
|-+-
|
124
|
-
|-+-
|
125
|
-
\-+-
|
159
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
160
|
+
|-+- 01049 ahoward ruby -Ilib samples/c.rb
|
161
|
+
|-+- 01050 ahoward ruby -Ilib samples/c.rb
|
162
|
+
\-+- 01051 ahoward ruby -Ilib samples/c.rb
|
126
163
|
|
127
164
|
---
|
128
165
|
b: |
|
129
|
-
-+-
|
130
|
-
|-+-
|
131
|
-
|-+-
|
132
|
-
\-+-
|
166
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
167
|
+
|-+- 01049 ahoward (ruby)
|
168
|
+
|-+- 01050 ahoward ruby -Ilib samples/c.rb
|
169
|
+
\-+- 01051 ahoward ruby -Ilib samples/c.rb
|
133
170
|
|
134
171
|
---
|
135
172
|
c: |
|
136
|
-
-+-
|
137
|
-
|-+-
|
138
|
-
|-+-
|
139
|
-
\-+-
|
173
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
174
|
+
|-+- 01049 ahoward ruby -Ilib samples/c.rb
|
175
|
+
|-+- 01050 ahoward ruby -Ilib samples/c.rb
|
176
|
+
\-+- 01051 ahoward ruby -Ilib samples/c.rb
|
140
177
|
|
141
178
|
---
|
142
179
|
d: |
|
143
|
-
-+-
|
144
|
-
|-+-
|
145
|
-
|-+-
|
146
|
-
|
180
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
181
|
+
|-+- 01061 ahoward ruby -Ilib samples/c.rb
|
182
|
+
|-+- 01062 ahoward ruby -Ilib samples/c.rb
|
183
|
+
\-+- 01063 ahoward ruby -Ilib samples/c.rb
|
147
184
|
|
148
185
|
---
|
149
186
|
e: |
|
150
|
-
-+-
|
151
|
-
|-+-
|
152
|
-
|-+-
|
153
|
-
|
187
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
188
|
+
|-+- 01061 ahoward (ruby)
|
189
|
+
|-+- 01062 ahoward ruby -Ilib samples/c.rb
|
190
|
+
\-+- 01063 ahoward ruby -Ilib samples/c.rb
|
154
191
|
|
155
192
|
---
|
156
193
|
f: |
|
157
|
-
-+-
|
158
|
-
|
159
|
-
|-+-
|
160
|
-
|
194
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
195
|
+
|-+- 01061 ahoward ruby -Ilib samples/c.rb
|
196
|
+
|-+- 01062 ahoward ruby -Ilib samples/c.rb
|
197
|
+
\-+- 01063 ahoward ruby -Ilib samples/c.rb
|
161
198
|
|
162
199
|
---
|
163
200
|
g: |
|
164
|
-
-+-
|
165
|
-
|
166
|
-
|-+-
|
167
|
-
|
201
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
202
|
+
|-+- 01090 ahoward ruby -Ilib samples/c.rb
|
203
|
+
|-+- 01091 ahoward ruby -Ilib samples/c.rb
|
204
|
+
\-+- 01092 ahoward ruby -Ilib samples/c.rb
|
168
205
|
|
169
206
|
---
|
170
207
|
h: |
|
171
|
-
-+-
|
172
|
-
|-+-
|
173
|
-
|
174
|
-
|
208
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
209
|
+
|-+- 01090 ahoward ruby -Ilib samples/c.rb
|
210
|
+
|-+- 01091 ahoward ruby -Ilib samples/c.rb
|
211
|
+
\-+- 01092 ahoward ruby -Ilib samples/c.rb
|
175
212
|
|
176
213
|
---
|
177
214
|
i: |
|
178
|
-
-+-
|
179
|
-
|-+-
|
180
|
-
|-+-
|
181
|
-
\-+-
|
215
|
+
-+- 01048 ahoward ruby -Ilib samples/c.rb
|
216
|
+
|-+- 01090 ahoward ruby -Ilib samples/c.rb
|
217
|
+
|-+- 01091 ahoward ruby -Ilib samples/c.rb
|
218
|
+
\-+- 01092 ahoward ruby -Ilib samples/c.rb
|
219
|
+
|
220
|
+
|
221
|
+
|
222
|
+
<========< samples/d.rb >========>
|
223
|
+
|
224
|
+
~ > cat samples/d.rb
|
225
|
+
|
226
|
+
# forkoff supports two strategies of reading the result from the child: via
|
227
|
+
# pipe (the default) or via file. you can select which to use using the
|
228
|
+
# :strategy option.
|
229
|
+
#
|
230
|
+
|
231
|
+
require 'forkoff'
|
182
232
|
|
233
|
+
%w( hey you guys ).forkoff :strategy => :file do |word|
|
234
|
+
puts "#{ word } from #{ Process.pid }"
|
235
|
+
end
|
236
|
+
|
237
|
+
~ > ruby samples/d.rb
|
238
|
+
|
239
|
+
hey from 1102
|
240
|
+
you from 1103
|
241
|
+
guys from 1104
|
183
242
|
|
data/gemspec.rb
CHANGED
@@ -21,7 +21,6 @@ Gem::Specification::new do |spec|
|
|
21
21
|
spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
|
22
22
|
|
23
23
|
spec.require_path = "lib"
|
24
|
-
spec.autorequire = lib
|
25
24
|
|
26
25
|
spec.has_rdoc = File::exist? "doc"
|
27
26
|
spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
|
@@ -32,4 +31,5 @@ Gem::Specification::new do |spec|
|
|
32
31
|
spec.author = "Ara T. Howard"
|
33
32
|
spec.email = "ara.t.howard@gmail.com"
|
34
33
|
spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
|
34
|
+
spec.rubyforge_project = 'codeforpeople'
|
35
35
|
end
|
data/install.rb
CHANGED
File without changes
|
data/lib/forkoff.rb
CHANGED
@@ -1,26 +1,142 @@
|
|
1
1
|
require 'thread'
|
2
2
|
|
3
3
|
module Forkoff
|
4
|
-
|
5
|
-
|
4
|
+
def version
|
5
|
+
'0.0.4'
|
6
|
+
end
|
7
|
+
|
8
|
+
def done
|
9
|
+
@done ||= Object.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def default
|
13
|
+
@default ||= { 'processes' => 2 }
|
14
|
+
end
|
15
|
+
|
16
|
+
def pipe
|
17
|
+
'pipe'
|
18
|
+
end
|
19
|
+
|
20
|
+
def file
|
21
|
+
'file'
|
22
|
+
end
|
23
|
+
|
24
|
+
def pid
|
25
|
+
@pid ||= Process.pid
|
26
|
+
end
|
27
|
+
|
28
|
+
def ppid
|
29
|
+
@ppid ||= Process.ppid
|
30
|
+
end
|
31
|
+
|
32
|
+
def tid
|
33
|
+
Thread.current.object_id.abs
|
34
|
+
end
|
35
|
+
|
36
|
+
def hostname
|
37
|
+
require 'socket'
|
38
|
+
@hostname ||= (Socket.gethostname rescue 'localhost.localdomain')
|
39
|
+
end
|
40
|
+
|
41
|
+
def tmpdir
|
42
|
+
require 'tmpdir'
|
43
|
+
@tmpdir ||= Dir.tmpdir
|
44
|
+
end
|
45
|
+
|
46
|
+
def tmpdir= tmpdir
|
47
|
+
@tmpdir = tmpdir.to_s
|
48
|
+
end
|
49
|
+
|
50
|
+
def tmpfile &block
|
51
|
+
basename = [hostname, pid, ppid, tid, rand].join('-')
|
52
|
+
tmp = File.join(tmpdir, basename)
|
53
|
+
|
54
|
+
fd = nil
|
55
|
+
flags = File::CREAT|File::EXCL|File::RDWR
|
56
|
+
|
57
|
+
42.times do
|
58
|
+
begin
|
59
|
+
fd = open tmp, flags
|
60
|
+
break
|
61
|
+
rescue Object
|
62
|
+
sleep rand
|
63
|
+
end
|
64
|
+
end
|
65
|
+
raise Error, "could not create tmpfile" unless fd
|
66
|
+
|
67
|
+
if block
|
68
|
+
begin
|
69
|
+
return block.call(fd)
|
70
|
+
ensure
|
71
|
+
fd.close unless fd.closed? rescue nil
|
72
|
+
FileUtils.rm_rf tmp rescue nil
|
73
|
+
end
|
74
|
+
else
|
75
|
+
return fd
|
76
|
+
end
|
77
|
+
end
|
6
78
|
|
7
|
-
|
8
|
-
|
79
|
+
def pipe_result *args, &block
|
80
|
+
r, w = IO.pipe
|
81
|
+
pid = fork
|
82
|
+
|
83
|
+
unless pid
|
84
|
+
r.close
|
85
|
+
result =
|
86
|
+
begin
|
87
|
+
block.call(*args)
|
88
|
+
rescue Object => e
|
89
|
+
e
|
90
|
+
end
|
91
|
+
w.write( Marshal.dump( result ) )
|
92
|
+
w.close
|
93
|
+
exit
|
94
|
+
end
|
95
|
+
|
96
|
+
w.close
|
97
|
+
data = ''
|
98
|
+
while(( buf = r.read(8192) ))
|
99
|
+
data << buf
|
100
|
+
end
|
101
|
+
result = Marshal.load( data )
|
102
|
+
r.close
|
103
|
+
Process.waitpid pid
|
104
|
+
return result
|
105
|
+
end
|
106
|
+
|
107
|
+
def file_result *args, &block
|
108
|
+
tmpfile do |fd|
|
109
|
+
pid = fork
|
9
110
|
|
10
|
-
|
11
|
-
|
12
|
-
|
111
|
+
unless pid
|
112
|
+
result =
|
113
|
+
begin
|
114
|
+
block.call(*args)
|
115
|
+
rescue Object => e
|
116
|
+
e
|
117
|
+
end
|
118
|
+
fd.write( Marshal.dump( result ) )
|
119
|
+
exit
|
120
|
+
end
|
13
121
|
|
14
|
-
|
15
|
-
|
122
|
+
Process.waitpid pid
|
123
|
+
fd.rewind
|
124
|
+
data = fd.read
|
125
|
+
result = Marshal.load( data )
|
126
|
+
return result
|
127
|
+
end
|
16
128
|
end
|
129
|
+
|
130
|
+
class Error < ::StandardError; end
|
131
|
+
|
132
|
+
extend self
|
17
133
|
end
|
18
134
|
|
19
135
|
module Enumerable
|
20
|
-
|
21
136
|
def forkoff options = {}, &block
|
22
137
|
options = { 'processes' => Integer(options) } unless Hash === options
|
23
138
|
n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
|
139
|
+
strategy = options['strategy'] || options[:strategy] || 'pipe'
|
24
140
|
qs = Array.new(n){ SizedQueue.new 1 }
|
25
141
|
results = Array.new(n){ [] }
|
26
142
|
|
@@ -39,25 +155,17 @@ module Enumerable
|
|
39
155
|
break if value == Forkoff.done
|
40
156
|
args, index = value
|
41
157
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
end
|
53
|
-
w.write( Marshal.dump( result ) )
|
54
|
-
exit
|
55
|
-
end
|
56
|
-
|
57
|
-
w.close
|
58
|
-
result = Marshal.load( r.read )
|
158
|
+
result =
|
159
|
+
case strategy.to_s.strip.downcase
|
160
|
+
when 'pipe'
|
161
|
+
Forkoff.pipe_result(*args, &block)
|
162
|
+
when 'file'
|
163
|
+
Forkoff.file_result(*args, &block)
|
164
|
+
else
|
165
|
+
raise ArgumentError, "strategy=#{ strategy.class }(#{ strategy.inspect })"
|
166
|
+
end
|
167
|
+
|
59
168
|
results[i].push( [result, index] )
|
60
|
-
Process.waitpid pid
|
61
169
|
end
|
62
170
|
|
63
171
|
results[i].push( Forkoff.done )
|
data/samples/a.rb
CHANGED
data/samples/b.rb
CHANGED
data/samples/c.rb
CHANGED
data/samples/d.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
# forkoff supports two strategies of reading the result from the child: via
|
2
|
+
# pipe (the default) or via file. you can select which to use using the
|
3
|
+
# :strategy option.
|
4
|
+
#
|
5
|
+
|
6
|
+
require 'forkoff'
|
7
|
+
|
8
|
+
%w( hey you guys ).forkoff :strategy => :file do |word|
|
9
|
+
puts "#{ word } from #{ Process.pid }"
|
10
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forkoff
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ara T. Howard
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-
|
12
|
+
date: 2008-09-13 00:00:00 -06:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -24,7 +24,6 @@ extra_rdoc_files: []
|
|
24
24
|
files:
|
25
25
|
- gemspec.rb
|
26
26
|
- gen_readme.rb
|
27
|
-
- HISTORY
|
28
27
|
- install.rb
|
29
28
|
- lib
|
30
29
|
- lib/forkoff.rb
|
@@ -33,6 +32,7 @@ files:
|
|
33
32
|
- samples/a.rb
|
34
33
|
- samples/b.rb
|
35
34
|
- samples/c.rb
|
35
|
+
- samples/d.rb
|
36
36
|
has_rdoc: false
|
37
37
|
homepage: http://codeforpeople.com/lib/ruby/forkoff/
|
38
38
|
post_install_message:
|
@@ -54,8 +54,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
54
54
|
version:
|
55
55
|
requirements: []
|
56
56
|
|
57
|
-
rubyforge_project:
|
58
|
-
rubygems_version: 1.0
|
57
|
+
rubyforge_project: codeforpeople
|
58
|
+
rubygems_version: 1.2.0
|
59
59
|
signing_key:
|
60
60
|
specification_version: 2
|
61
61
|
summary: forkoff
|
data/HISTORY
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
|
2
|
-
0.0.1
|
3
|
-
|
4
|
-
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
5
|
-
channel. in this way the producers do not build up a massize parllel data
|
6
|
-
structure but provide data to the consumers only as fast as they can fork
|
7
|
-
and proccess it. basically for a 4 process run you'll end up with 4
|
8
|
-
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
9
|
-
thread popping of jobs, forking, and yielding results.
|
10
|
-
|
11
|
-
- removed use of Queue for capturing the output. now it's simply an array
|
12
|
-
of arrays which removed some sync overhead.
|
13
|
-
|
14
|
-
- you can configure the number of processes globally with
|
15
|
-
|
16
|
-
Forkoff.default['proccess'] = 4
|
17
|
-
|
18
|
-
- you can now pass either an options hash
|
19
|
-
|
20
|
-
forkoff( :processes => 2 ) ...
|
21
|
-
|
22
|
-
or plain vanilla number
|
23
|
-
|
24
|
-
forkoff( 2 ) ...
|
25
|
-
|
26
|
-
to the forkoff call
|
27
|
-
|
28
|
-
- default number of processes is 8, not 2
|
29
|
-
|
30
|
-
|
31
|
-
0.0.0
|
32
|
-
|
33
|
-
initial version
|
34
|
-
|