forkoff 0.0.4 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README +96 -84
- data/forkoff.gemspec +27 -0
- data/lib/forkoff.rb +17 -15
- data/rakefile +223 -0
- data/readme.erb +76 -0
- data/test/forkoff.rb +54 -0
- metadata +15 -14
- data/gemspec.rb +0 -35
- data/gen_readme.rb +0 -32
- data/install.rb +0 -214
data/README
CHANGED
@@ -9,6 +9,7 @@ SYNOPSIS
|
|
9
9
|
URI
|
10
10
|
|
11
11
|
http://rubyforge.org/projects/codeforpeople
|
12
|
+
http://github.com/ahoward/forkoff
|
12
13
|
|
13
14
|
INSTALL
|
14
15
|
|
@@ -20,48 +21,9 @@ DESCRIPTION
|
|
20
21
|
child process and collecting the results. forkoff can limit the number of
|
21
22
|
child processes which is, by default, 2.
|
22
23
|
|
23
|
-
HISTORY
|
24
|
-
0.0.4
|
25
|
-
- code re-org
|
26
|
-
- add :strategy option
|
27
|
-
- default number of processes is 2, not 8
|
28
|
-
|
29
|
-
0.0.1
|
30
|
-
|
31
|
-
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
32
|
-
channel. in this way the producers do not build up a massize parllel data
|
33
|
-
structure but provide data to the consumers only as fast as they can fork
|
34
|
-
and proccess it. basically for a 4 process run you'll end up with 4
|
35
|
-
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
36
|
-
thread popping of jobs, forking, and yielding results.
|
37
|
-
|
38
|
-
- removed use of Queue for capturing the output. now it's simply an array
|
39
|
-
of arrays which removed some sync overhead.
|
40
|
-
|
41
|
-
- you can configure the number of processes globally with
|
42
|
-
|
43
|
-
Forkoff.default['proccess'] = 4
|
44
|
-
|
45
|
-
- you can now pass either an options hash
|
46
|
-
|
47
|
-
forkoff( :processes => 2 ) ...
|
48
|
-
|
49
|
-
or plain vanilla number
|
50
|
-
|
51
|
-
forkoff( 2 ) ...
|
52
|
-
|
53
|
-
to the forkoff call
|
54
|
-
|
55
|
-
- default number of processes is 8, not 2
|
56
|
-
|
57
|
-
|
58
|
-
0.0.0
|
59
|
-
|
60
|
-
initial version
|
61
|
-
|
62
|
-
|
63
24
|
SAMPLES
|
64
25
|
|
26
|
+
|
65
27
|
<========< samples/a.rb >========>
|
66
28
|
|
67
29
|
~ > cat samples/a.rb
|
@@ -76,8 +38,8 @@ SAMPLES
|
|
76
38
|
|
77
39
|
~ > ruby samples/a.rb
|
78
40
|
|
79
|
-
hey from
|
80
|
-
you from
|
41
|
+
hey from 7907
|
42
|
+
you from 7908
|
81
43
|
|
82
44
|
|
83
45
|
<========< samples/b.rb >========>
|
@@ -107,7 +69,7 @@ SAMPLES
|
|
107
69
|
|
108
70
|
~ > ruby samples/b.rb
|
109
71
|
|
110
|
-
elapsed: 4.
|
72
|
+
elapsed: 4.19184589385986
|
111
73
|
results: [0, 1, 4, 9, 16, 25, 36, 49]
|
112
74
|
|
113
75
|
|
@@ -151,71 +113,70 @@ SAMPLES
|
|
151
113
|
~ > ruby samples/c.rb
|
152
114
|
|
153
115
|
|
154
|
-
pid:
|
155
|
-
elapsed: 3.
|
116
|
+
pid: 7922
|
117
|
+
elapsed: 3.37899208068848
|
156
118
|
|
157
119
|
---
|
158
120
|
a: |
|
159
|
-
-+-
|
160
|
-
|-+-
|
161
|
-
|-+-
|
162
|
-
\-+-
|
121
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
122
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
123
|
+
|-+- 07924 ahoward (ruby)
|
124
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
163
125
|
|
164
126
|
---
|
165
127
|
b: |
|
166
|
-
-+-
|
167
|
-
|-+-
|
168
|
-
|-+-
|
169
|
-
\-+-
|
128
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
129
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
130
|
+
|-+- 07924 ahoward ruby -Ilib samples/c.rb
|
131
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
170
132
|
|
171
133
|
---
|
172
134
|
c: |
|
173
|
-
-+-
|
174
|
-
|-+-
|
175
|
-
|-+-
|
176
|
-
\-+-
|
135
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
136
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
137
|
+
|-+- 07924 ahoward (ruby)
|
138
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
177
139
|
|
178
140
|
---
|
179
141
|
d: |
|
180
|
-
-+-
|
181
|
-
|-+-
|
182
|
-
|
183
|
-
|
142
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
143
|
+
|-+- 07932 ahoward ruby -Ilib samples/c.rb
|
144
|
+
|--- 07933 ahoward ruby -Ilib samples/c.rb
|
145
|
+
\--- 07934 ahoward ruby -Ilib samples/c.rb
|
184
146
|
|
185
147
|
---
|
186
148
|
e: |
|
187
|
-
-+-
|
188
|
-
|
189
|
-
|-+-
|
190
|
-
\-+-
|
149
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
150
|
+
|--- 07932 ahoward (ruby)
|
151
|
+
|-+- 07933 ahoward ruby -Ilib samples/c.rb
|
152
|
+
\-+- 07934 ahoward (ruby)
|
191
153
|
|
192
154
|
---
|
193
155
|
f: |
|
194
|
-
-+-
|
195
|
-
|
196
|
-
|-+-
|
197
|
-
\-+-
|
156
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
157
|
+
|--- 07932 ahoward (ruby)
|
158
|
+
|-+- 07933 ahoward ruby -Ilib samples/c.rb
|
159
|
+
\-+- 07934 ahoward ruby -Ilib samples/c.rb
|
198
160
|
|
199
161
|
---
|
200
162
|
g: |
|
201
|
-
-+-
|
202
|
-
|-+-
|
203
|
-
|
204
|
-
|
163
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
164
|
+
|-+- 07941 ahoward ruby -Ilib samples/c.rb
|
165
|
+
|--- 07942 ahoward ruby -Ilib samples/c.rb
|
166
|
+
\--- 07943 ahoward ruby -Ilib samples/c.rb
|
205
167
|
|
206
168
|
---
|
207
169
|
h: |
|
208
|
-
-+-
|
209
|
-
|-+-
|
210
|
-
|-+-
|
211
|
-
|
170
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
171
|
+
|-+- 07941 ahoward (ruby)
|
172
|
+
|-+- 07942 ahoward ruby -Ilib samples/c.rb
|
173
|
+
\--- 07943 ahoward ruby -Ilib samples/c.rb
|
212
174
|
|
213
175
|
---
|
214
176
|
i: |
|
215
|
-
-+-
|
216
|
-
|
217
|
-
|
218
|
-
\-+- 01092 ahoward ruby -Ilib samples/c.rb
|
177
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
178
|
+
|--- 07942 ahoward (ruby)
|
179
|
+
\-+- 07943 ahoward ruby -Ilib samples/c.rb
|
219
180
|
|
220
181
|
|
221
182
|
|
@@ -236,7 +197,58 @@ SAMPLES
|
|
236
197
|
|
237
198
|
~ > ruby samples/d.rb
|
238
199
|
|
239
|
-
hey from
|
240
|
-
you from
|
241
|
-
guys from
|
200
|
+
hey from 7953
|
201
|
+
you from 7954
|
202
|
+
guys from 7955
|
203
|
+
|
242
204
|
|
205
|
+
|
206
|
+
HISTORY
|
207
|
+
1.1.0
|
208
|
+
- move to a model with one work queue and signals sent from consumers to
|
209
|
+
producer to noitify ready state. this let's smaller jobs race through a
|
210
|
+
single process even while a larger job may have one sub-process bound up.
|
211
|
+
incorporates a fix from http://github.com/fredrikj/forkoff which meant
|
212
|
+
some processes would lag behind when jobs didn't have similar execution
|
213
|
+
times.
|
214
|
+
|
215
|
+
1.0.0
|
216
|
+
- move to github
|
217
|
+
|
218
|
+
0.0.4
|
219
|
+
- code re-org
|
220
|
+
- add :strategy option
|
221
|
+
- default number of processes is 2, not 8
|
222
|
+
|
223
|
+
0.0.1
|
224
|
+
|
225
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
226
|
+
channel. in this way the producers do not build up a massize parllel data
|
227
|
+
structure but provide data to the consumers only as fast as they can fork
|
228
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
229
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
230
|
+
thread popping of jobs, forking, and yielding results.
|
231
|
+
|
232
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
233
|
+
of arrays which removed some sync overhead.
|
234
|
+
|
235
|
+
- you can configure the number of processes globally with
|
236
|
+
|
237
|
+
Forkoff.default['proccess'] = 4
|
238
|
+
|
239
|
+
- you can now pass either an options hash
|
240
|
+
|
241
|
+
forkoff( :processes => 2 ) ...
|
242
|
+
|
243
|
+
or plain vanilla number
|
244
|
+
|
245
|
+
forkoff( 2 ) ...
|
246
|
+
|
247
|
+
to the forkoff call
|
248
|
+
|
249
|
+
- default number of processes is 8, not 2
|
250
|
+
|
251
|
+
|
252
|
+
0.0.0
|
253
|
+
|
254
|
+
initial version
|
data/forkoff.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
## forkoff.gemspec
|
2
|
+
#
|
3
|
+
|
4
|
+
Gem::Specification::new do |spec|
|
5
|
+
spec.name = "forkoff"
|
6
|
+
spec.description = 'brain-dead simple parallel processing for ruby'
|
7
|
+
spec.version = "1.1.0"
|
8
|
+
spec.platform = Gem::Platform::RUBY
|
9
|
+
spec.summary = "forkoff"
|
10
|
+
|
11
|
+
spec.files = ["forkoff.gemspec", "lib", "lib/forkoff.rb", "rakefile", "README", "readme.erb", "samples", "samples/a.rb", "samples/b.rb", "samples/c.rb", "samples/d.rb", "test", "test/forkoff.rb"]
|
12
|
+
spec.executables = []
|
13
|
+
|
14
|
+
spec.require_path = "lib"
|
15
|
+
|
16
|
+
spec.has_rdoc = true
|
17
|
+
spec.test_files = "test/forkoff.rb"
|
18
|
+
#spec.add_dependency 'lib', '>= version'
|
19
|
+
#spec.add_dependency 'fattr'
|
20
|
+
|
21
|
+
spec.extensions.push(*[])
|
22
|
+
|
23
|
+
spec.rubyforge_project = "codeforpeople"
|
24
|
+
spec.author = "Ara T. Howard"
|
25
|
+
spec.email = "ara.t.howard@gmail.com"
|
26
|
+
spec.homepage = "http://github.com/ahoward/forkoff/tree/master"
|
27
|
+
end
|
data/lib/forkoff.rb
CHANGED
@@ -2,13 +2,17 @@ require 'thread'
|
|
2
2
|
|
3
3
|
module Forkoff
|
4
4
|
def version
|
5
|
-
'
|
5
|
+
'1.1.0'
|
6
6
|
end
|
7
7
|
|
8
8
|
def done
|
9
9
|
@done ||= Object.new
|
10
10
|
end
|
11
11
|
|
12
|
+
def ready
|
13
|
+
@ready ||= Object.new
|
14
|
+
end
|
15
|
+
|
12
16
|
def default
|
13
17
|
@default ||= { 'processes' => 2 }
|
14
18
|
end
|
@@ -137,7 +141,8 @@ module Enumerable
|
|
137
141
|
options = { 'processes' => Integer(options) } unless Hash === options
|
138
142
|
n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
|
139
143
|
strategy = options['strategy'] || options[:strategy] || 'pipe'
|
140
|
-
|
144
|
+
p2c = Queue.new
|
145
|
+
c2p = Queue.new
|
141
146
|
results = Array.new(n){ [] }
|
142
147
|
|
143
148
|
#
|
@@ -151,7 +156,8 @@ module Enumerable
|
|
151
156
|
Thread.current.abort_on_exception = true
|
152
157
|
|
153
158
|
loop do
|
154
|
-
|
159
|
+
c2p.push( Forkoff.ready )
|
160
|
+
value = p2c.pop
|
155
161
|
break if value == Forkoff.done
|
156
162
|
args, index = value
|
157
163
|
|
@@ -177,22 +183,18 @@ module Enumerable
|
|
177
183
|
#
|
178
184
|
# producers
|
179
185
|
#
|
180
|
-
|
181
|
-
|
182
|
-
n.times do |i|
|
183
|
-
thread = Thread.new do
|
186
|
+
producer =
|
187
|
+
Thread.new do
|
184
188
|
Thread.current.abort_on_exception = true
|
185
|
-
each_with_index do |args,
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
+
each_with_index do |args, i|
|
190
|
+
ready = c2p.pop
|
191
|
+
p2c.push( [args, i] )
|
192
|
+
end
|
193
|
+
n.times do |i|
|
194
|
+
p2c.push( Forkoff.done )
|
189
195
|
end
|
190
|
-
qs[ i ].push( Forkoff.done )
|
191
196
|
end
|
192
197
|
|
193
|
-
producers << thread
|
194
|
-
end
|
195
|
-
|
196
198
|
#
|
197
199
|
# wait for all consumers to complete
|
198
200
|
#
|
data/rakefile
ADDED
@@ -0,0 +1,223 @@
|
|
1
|
+
|
2
|
+
This.rubyforge_project = 'codeforpeople'
|
3
|
+
This.author = "Ara T. Howard"
|
4
|
+
This.email = "ara.t.howard@gmail.com"
|
5
|
+
This.homepage = "http://github.com/ahoward/#{ This.lib }/tree/master"
|
6
|
+
|
7
|
+
|
8
|
+
task :default do
|
9
|
+
puts(Rake::Task.tasks.map{|task| task.name} - ['default'])
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
task :gemspec do
|
14
|
+
ignore_extensions = 'git', 'svn', 'tmp', /sw./, 'bak', 'gem'
|
15
|
+
ignore_directories = 'pkg'
|
16
|
+
|
17
|
+
shiteless =
|
18
|
+
lambda do |list|
|
19
|
+
list.delete_if do |entry|
|
20
|
+
next unless test(?e, entry)
|
21
|
+
extension = File.basename(entry).split(%r/[.]/).last
|
22
|
+
ignore_extensions.any?{|ext| ext === extension}
|
23
|
+
end
|
24
|
+
list.delete_if do |entry|
|
25
|
+
next unless test(?d, entry)
|
26
|
+
dirname = File.expand_path(entry)
|
27
|
+
ignore_directories.any?{|dir| File.expand_path(dir) == dirname}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
lib = This.lib
|
32
|
+
version = This.version
|
33
|
+
files = shiteless[Dir::glob("**/**")]
|
34
|
+
executables = shiteless[Dir::glob("bin/*")].map{|exe| File.basename(exe)}
|
35
|
+
has_rdoc = true #File.exist?('doc')
|
36
|
+
test_files = "test/#{ lib }.rb" if File.file?("test/#{ lib }.rb")
|
37
|
+
|
38
|
+
extensions = This.extensions
|
39
|
+
if extensions.nil?
|
40
|
+
%w( Makefile configure extconf.rb ).each do |ext|
|
41
|
+
extensions << ext if File.exists?(ext)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
extensions = [extensions].flatten.compact
|
45
|
+
|
46
|
+
template =
|
47
|
+
if test(?e, 'gemspec.erb')
|
48
|
+
Template{ IO.read('gemspec.erb') }
|
49
|
+
else
|
50
|
+
Template {
|
51
|
+
<<-__
|
52
|
+
## #{ lib }.gemspec
|
53
|
+
#
|
54
|
+
|
55
|
+
Gem::Specification::new do |spec|
|
56
|
+
spec.name = #{ lib.inspect }
|
57
|
+
spec.description = 'brain-dead simple parallel processing for ruby'
|
58
|
+
spec.version = #{ version.inspect }
|
59
|
+
spec.platform = Gem::Platform::RUBY
|
60
|
+
spec.summary = #{ lib.inspect }
|
61
|
+
|
62
|
+
spec.files = #{ files.inspect }
|
63
|
+
spec.executables = #{ executables.inspect }
|
64
|
+
|
65
|
+
spec.require_path = "lib"
|
66
|
+
|
67
|
+
spec.has_rdoc = #{ has_rdoc.inspect }
|
68
|
+
spec.test_files = #{ test_files.inspect }
|
69
|
+
#spec.add_dependency 'lib', '>= version'
|
70
|
+
#spec.add_dependency 'fattr'
|
71
|
+
|
72
|
+
spec.extensions.push(*#{ extensions.inspect })
|
73
|
+
|
74
|
+
spec.rubyforge_project = #{ This.rubyforge_project.inspect }
|
75
|
+
spec.author = #{ This.author.inspect }
|
76
|
+
spec.email = #{ This.email.inspect }
|
77
|
+
spec.homepage = #{ This.homepage.inspect }
|
78
|
+
end
|
79
|
+
__
|
80
|
+
}
|
81
|
+
end
|
82
|
+
|
83
|
+
open("#{ lib }.gemspec", "w"){|fd| fd.puts template}
|
84
|
+
This.gemspec = "#{ lib }.gemspec"
|
85
|
+
end
|
86
|
+
|
87
|
+
task :gem => [:clean, :gemspec] do
|
88
|
+
Fu.mkdir_p This.pkgdir
|
89
|
+
before = Dir['*.gem']
|
90
|
+
cmd = "gem build #{ This.gemspec }"
|
91
|
+
`#{ cmd }`
|
92
|
+
after = Dir['*.gem']
|
93
|
+
gem = ((after - before).first || after.first) or abort('no gem!')
|
94
|
+
Fu.mv gem, This.pkgdir
|
95
|
+
This.gem = File.basename(gem)
|
96
|
+
end
|
97
|
+
|
98
|
+
task :readme do
|
99
|
+
samples = ''
|
100
|
+
prompt = '~ > '
|
101
|
+
lib = This.lib
|
102
|
+
version = This.version
|
103
|
+
|
104
|
+
Dir['sample*/*'].sort.each do |sample|
|
105
|
+
samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
|
106
|
+
|
107
|
+
cmd = "cat #{ sample }"
|
108
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
109
|
+
samples << Util.indent(`#{ cmd }`, 4) << "\n"
|
110
|
+
|
111
|
+
cmd = "ruby #{ sample }"
|
112
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
113
|
+
|
114
|
+
cmd = "ruby -e'STDOUT.sync=true; exec %(ruby -Ilib #{ sample })'"
|
115
|
+
samples << Util.indent(`#{ cmd } 2>&1`, 4) << "\n"
|
116
|
+
end
|
117
|
+
|
118
|
+
template =
|
119
|
+
if test(?e, 'readme.erb')
|
120
|
+
Template{ IO.read('readme.erb') }
|
121
|
+
else
|
122
|
+
Template {
|
123
|
+
<<-__
|
124
|
+
NAME
|
125
|
+
#{ lib }
|
126
|
+
|
127
|
+
DESCRIPTION
|
128
|
+
|
129
|
+
INSTALL
|
130
|
+
gem install #{ lib }
|
131
|
+
|
132
|
+
SAMPLES
|
133
|
+
#{ samples }
|
134
|
+
__
|
135
|
+
}
|
136
|
+
end
|
137
|
+
|
138
|
+
open("README", "w"){|fd| fd.puts template}
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
task :clean do
|
143
|
+
Dir[File.join(This.pkgdir, '**/**')].each{|entry| Fu.rm_rf(entry)}
|
144
|
+
end
|
145
|
+
|
146
|
+
|
147
|
+
task :release => [:clean, :gemspec, :gem] do
|
148
|
+
gems = Dir[File.join(This.pkgdir, '*.gem')].flatten
|
149
|
+
raise "which one? : #{ gems.inspect }" if gems.size > 1
|
150
|
+
raise "no gems?" if gems.size < 1
|
151
|
+
cmd = "rubyforge login && rubyforge add_release #{ This.rubyforge_project } #{ This.lib } #{ This.version } #{ This.pkgdir }/#{ This.gem }"
|
152
|
+
puts cmd
|
153
|
+
system cmd
|
154
|
+
end
|
155
|
+
|
156
|
+
|
157
|
+
|
158
|
+
|
159
|
+
|
160
|
+
BEGIN {
|
161
|
+
$VERBOSE = nil
|
162
|
+
|
163
|
+
require 'ostruct'
|
164
|
+
require 'erb'
|
165
|
+
require 'fileutils'
|
166
|
+
|
167
|
+
Fu = FileUtils
|
168
|
+
|
169
|
+
This = OpenStruct.new
|
170
|
+
|
171
|
+
This.file = File.expand_path(__FILE__)
|
172
|
+
This.dir = File.dirname(This.file)
|
173
|
+
This.pkgdir = File.join(This.dir, 'pkg')
|
174
|
+
|
175
|
+
lib = ENV['LIB']
|
176
|
+
unless lib
|
177
|
+
lib = File.basename(Dir.pwd)
|
178
|
+
end
|
179
|
+
This.lib = lib
|
180
|
+
|
181
|
+
version = ENV['VERSION']
|
182
|
+
unless version
|
183
|
+
name = lib.capitalize
|
184
|
+
require "./lib/#{ lib }"
|
185
|
+
version = eval(name).send(:version)
|
186
|
+
end
|
187
|
+
This.version = version
|
188
|
+
|
189
|
+
abort('no lib') unless This.lib
|
190
|
+
abort('no version') unless This.version
|
191
|
+
|
192
|
+
module Util
|
193
|
+
def indent(s, n = 2)
|
194
|
+
s = unindent(s)
|
195
|
+
ws = ' ' * n
|
196
|
+
s.gsub(%r/^/, ws)
|
197
|
+
end
|
198
|
+
|
199
|
+
def unindent(s)
|
200
|
+
indent = nil
|
201
|
+
s.each do |line|
|
202
|
+
next if line =~ %r/^\s*$/
|
203
|
+
indent = line[%r/^\s*/] and break
|
204
|
+
end
|
205
|
+
indent ? s.gsub(%r/^#{ indent }/, "") : s
|
206
|
+
end
|
207
|
+
extend self
|
208
|
+
end
|
209
|
+
|
210
|
+
class Template
|
211
|
+
def initialize(&block)
|
212
|
+
@block = block
|
213
|
+
@template = block.call.to_s
|
214
|
+
end
|
215
|
+
def expand(b=nil)
|
216
|
+
ERB.new(Util.unindent(@template)).result(b||@block)
|
217
|
+
end
|
218
|
+
alias_method 'to_s', 'expand'
|
219
|
+
end
|
220
|
+
def Template(*args, &block) Template.new(*args, &block) end
|
221
|
+
|
222
|
+
Dir.chdir(This.dir)
|
223
|
+
}
|
data/readme.erb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
NAME
|
2
|
+
|
3
|
+
forkoff
|
4
|
+
|
5
|
+
SYNOPSIS
|
6
|
+
|
7
|
+
brain-dead simple parallel processing for ruby
|
8
|
+
|
9
|
+
URI
|
10
|
+
|
11
|
+
http://rubyforge.org/projects/codeforpeople
|
12
|
+
http://github.com/ahoward/forkoff
|
13
|
+
|
14
|
+
INSTALL
|
15
|
+
|
16
|
+
gem install forkoff
|
17
|
+
|
18
|
+
DESCRIPTION
|
19
|
+
|
20
|
+
forkoff works for any enumerable object, iterating a code block to run in a
|
21
|
+
child process and collecting the results. forkoff can limit the number of
|
22
|
+
child processes which is, by default, 2.
|
23
|
+
|
24
|
+
SAMPLES
|
25
|
+
|
26
|
+
<%= samples %>
|
27
|
+
|
28
|
+
HISTORY
|
29
|
+
1.1.0
|
30
|
+
- move to a model with one work queue and signals sent from consumers to
|
31
|
+
producer to noitify ready state. this let's smaller jobs race through a
|
32
|
+
single process even while a larger job may have one sub-process bound up.
|
33
|
+
incorporates a fix from http://github.com/fredrikj/forkoff which meant
|
34
|
+
some processes would lag behind when jobs didn't have similar execution
|
35
|
+
times.
|
36
|
+
|
37
|
+
1.0.0
|
38
|
+
- move to github
|
39
|
+
|
40
|
+
0.0.4
|
41
|
+
- code re-org
|
42
|
+
- add :strategy option
|
43
|
+
- default number of processes is 2, not 8
|
44
|
+
|
45
|
+
0.0.1
|
46
|
+
|
47
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
48
|
+
channel. in this way the producers do not build up a massize parllel data
|
49
|
+
structure but provide data to the consumers only as fast as they can fork
|
50
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
51
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
52
|
+
thread popping of jobs, forking, and yielding results.
|
53
|
+
|
54
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
55
|
+
of arrays which removed some sync overhead.
|
56
|
+
|
57
|
+
- you can configure the number of processes globally with
|
58
|
+
|
59
|
+
Forkoff.default['proccess'] = 4
|
60
|
+
|
61
|
+
- you can now pass either an options hash
|
62
|
+
|
63
|
+
forkoff( :processes => 2 ) ...
|
64
|
+
|
65
|
+
or plain vanilla number
|
66
|
+
|
67
|
+
forkoff( 2 ) ...
|
68
|
+
|
69
|
+
to the forkoff call
|
70
|
+
|
71
|
+
- default number of processes is 8, not 2
|
72
|
+
|
73
|
+
|
74
|
+
0.0.0
|
75
|
+
|
76
|
+
initial version
|
data/test/forkoff.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
$:.unshift('.')
|
2
|
+
$:.unshift('./lib')
|
3
|
+
$:.unshift('..')
|
4
|
+
$:.unshift('../lib')
|
5
|
+
|
6
|
+
require('test/unit')
|
7
|
+
require('forkoff')
|
8
|
+
|
9
|
+
|
10
|
+
class T < Test::Unit::TestCase
|
11
|
+
|
12
|
+
# simple usage
|
13
|
+
#
|
14
|
+
|
15
|
+
def test_0010
|
16
|
+
results = [0,1,2].forkoff!{|n| [n, Process.pid]}
|
17
|
+
assert_equal(results, results.uniq, 'all ran in a different process')
|
18
|
+
end
|
19
|
+
|
20
|
+
# it's faster
|
21
|
+
#
|
22
|
+
def test_0020
|
23
|
+
n = 4
|
24
|
+
strategies = :forkoff, :each
|
25
|
+
|
26
|
+
4.times do
|
27
|
+
result = {}
|
28
|
+
|
29
|
+
strategies.each do |strategy|
|
30
|
+
a = Time.now.to_f
|
31
|
+
(0..4).send(strategy){|i| sleep 0.2}
|
32
|
+
b = Time.now.to_f
|
33
|
+
elapsed = b - a
|
34
|
+
result[strategy] = elapsed
|
35
|
+
end
|
36
|
+
|
37
|
+
assert result[:forkoff] < result[:each], 'forkoff is faster than each for long running tasks'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# in case of different execution times for different processes
|
42
|
+
#
|
43
|
+
def test_0030
|
44
|
+
a = Time.now.to_f
|
45
|
+
(0...4).forkoff(2) do |i|
|
46
|
+
sleep i.modulo(2)
|
47
|
+
end
|
48
|
+
b = Time.now.to_f
|
49
|
+
elapsed = b - a
|
50
|
+
assert elapsed < 2
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forkoff
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ara T. Howard
|
@@ -9,11 +9,11 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2009-10-11 00:00:00 -06:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
16
|
-
description:
|
16
|
+
description: brain-dead simple parallel processing for ruby
|
17
17
|
email: ara.t.howard@gmail.com
|
18
18
|
executables: []
|
19
19
|
|
@@ -22,19 +22,20 @@ extensions: []
|
|
22
22
|
extra_rdoc_files: []
|
23
23
|
|
24
24
|
files:
|
25
|
-
- gemspec
|
26
|
-
- gen_readme.rb
|
27
|
-
- install.rb
|
28
|
-
- lib
|
25
|
+
- forkoff.gemspec
|
29
26
|
- lib/forkoff.rb
|
27
|
+
- rakefile
|
30
28
|
- README
|
31
|
-
-
|
29
|
+
- readme.erb
|
32
30
|
- samples/a.rb
|
33
31
|
- samples/b.rb
|
34
32
|
- samples/c.rb
|
35
33
|
- samples/d.rb
|
36
|
-
|
37
|
-
|
34
|
+
- test/forkoff.rb
|
35
|
+
has_rdoc: true
|
36
|
+
homepage: http://github.com/ahoward/forkoff/tree/master
|
37
|
+
licenses: []
|
38
|
+
|
38
39
|
post_install_message:
|
39
40
|
rdoc_options: []
|
40
41
|
|
@@ -55,9 +56,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
55
56
|
requirements: []
|
56
57
|
|
57
58
|
rubyforge_project: codeforpeople
|
58
|
-
rubygems_version: 1.
|
59
|
+
rubygems_version: 1.3.5
|
59
60
|
signing_key:
|
60
|
-
specification_version:
|
61
|
+
specification_version: 3
|
61
62
|
summary: forkoff
|
62
|
-
test_files:
|
63
|
-
|
63
|
+
test_files:
|
64
|
+
- test/forkoff.rb
|
data/gemspec.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
lib, version = File::basename(File::dirname(File::expand_path(__FILE__))).split %r/-/, 2
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
|
5
|
-
Gem::Specification::new do |spec|
|
6
|
-
$VERBOSE = nil
|
7
|
-
|
8
|
-
shiteless = lambda do |list|
|
9
|
-
list.delete_if do |file|
|
10
|
-
file =~ %r/\.svn/ or
|
11
|
-
file =~ %r/\.tmp/
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
spec.name = lib
|
16
|
-
spec.version = version
|
17
|
-
spec.platform = Gem::Platform::RUBY
|
18
|
-
spec.summary = lib
|
19
|
-
|
20
|
-
spec.files = shiteless[Dir::glob("**/**")]
|
21
|
-
spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
|
22
|
-
|
23
|
-
spec.require_path = "lib"
|
24
|
-
|
25
|
-
spec.has_rdoc = File::exist? "doc"
|
26
|
-
spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
|
27
|
-
#spec.add_dependency 'lib', '>= version'
|
28
|
-
|
29
|
-
spec.extensions << "extconf.rb" if File::exists? "extconf.rb"
|
30
|
-
|
31
|
-
spec.author = "Ara T. Howard"
|
32
|
-
spec.email = "ara.t.howard@gmail.com"
|
33
|
-
spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
|
34
|
-
spec.rubyforge_project = 'codeforpeople'
|
35
|
-
end
|
data/gen_readme.rb
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
require 'pathname'
|
2
|
-
|
3
|
-
$VERBOSE=nil
|
4
|
-
|
5
|
-
def indent s, n = 2
|
6
|
-
ws = ' ' * n
|
7
|
-
s.gsub %r/^/, ws
|
8
|
-
end
|
9
|
-
|
10
|
-
template = IO::read 'README.tmpl'
|
11
|
-
|
12
|
-
samples = ''
|
13
|
-
prompt = '~ > '
|
14
|
-
|
15
|
-
Dir['sample*/*'].sort.each do |sample|
|
16
|
-
samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
|
17
|
-
|
18
|
-
cmd = "cat #{ sample }"
|
19
|
-
samples << indent(prompt + cmd, 2) << "\n\n"
|
20
|
-
samples << indent(`#{ cmd }`, 4) << "\n"
|
21
|
-
|
22
|
-
cmd = "ruby #{ sample }"
|
23
|
-
samples << indent(prompt + cmd, 2) << "\n\n"
|
24
|
-
|
25
|
-
cmd = "ruby -Ilib #{ sample }"
|
26
|
-
samples << indent(`#{ cmd } 2>&1`, 4) << "\n"
|
27
|
-
end
|
28
|
-
|
29
|
-
#samples.gsub! %r/^/, ' '
|
30
|
-
|
31
|
-
readme = template.gsub %r/^\s*@samples\s*$/, samples
|
32
|
-
print readme
|
data/install.rb
DELETED
@@ -1,214 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
require 'rbconfig'
|
3
|
-
require 'find'
|
4
|
-
require 'ftools'
|
5
|
-
require 'tempfile'
|
6
|
-
include Config
|
7
|
-
|
8
|
-
LIBDIR = "lib"
|
9
|
-
LIBDIR_MODE = 0644
|
10
|
-
|
11
|
-
BINDIR = "bin"
|
12
|
-
BINDIR_MODE = 0755
|
13
|
-
|
14
|
-
|
15
|
-
$srcdir = CONFIG["srcdir"]
|
16
|
-
$version = CONFIG["MAJOR"]+"."+CONFIG["MINOR"]
|
17
|
-
$libdir = File.join(CONFIG["libdir"], "ruby", $version)
|
18
|
-
$archdir = File.join($libdir, CONFIG["arch"])
|
19
|
-
$site_libdir = $:.find {|x| x =~ /site_ruby$/}
|
20
|
-
$bindir = CONFIG["bindir"] || CONFIG['BINDIR']
|
21
|
-
$ruby_install_name = CONFIG['ruby_install_name'] || CONFIG['RUBY_INSTALL_NAME'] || 'ruby'
|
22
|
-
$ruby_ext = CONFIG['EXEEXT'] || ''
|
23
|
-
$ruby = File.join($bindir, ($ruby_install_name + $ruby_ext))
|
24
|
-
|
25
|
-
if !$site_libdir
|
26
|
-
$site_libdir = File.join($libdir, "site_ruby")
|
27
|
-
elsif $site_libdir !~ %r/#{Regexp.quote($version)}/
|
28
|
-
$site_libdir = File.join($site_libdir, $version)
|
29
|
-
end
|
30
|
-
|
31
|
-
def install_rb(srcdir=nil, destdir=nil, mode=nil, bin=nil)
|
32
|
-
#{{{
|
33
|
-
path = []
|
34
|
-
dir = []
|
35
|
-
Find.find(srcdir) do |f|
|
36
|
-
next unless FileTest.file?(f)
|
37
|
-
next if (f = f[srcdir.length+1..-1]) == nil
|
38
|
-
next if (/CVS$/ =~ File.dirname(f))
|
39
|
-
next if (/\.svn/ =~ File.dirname(f))
|
40
|
-
next if f =~ %r/\.lnk/
|
41
|
-
next if f =~ %r/\.svn/
|
42
|
-
next if f =~ %r/\.swp/
|
43
|
-
next if f =~ %r/\.svn/
|
44
|
-
path.push f
|
45
|
-
dir |= [File.dirname(f)]
|
46
|
-
end
|
47
|
-
for f in dir
|
48
|
-
next if f == "."
|
49
|
-
next if f == "CVS"
|
50
|
-
File::makedirs(File.join(destdir, f))
|
51
|
-
end
|
52
|
-
for f in path
|
53
|
-
next if (/\~$/ =~ f)
|
54
|
-
next if (/^\./ =~ File.basename(f))
|
55
|
-
unless bin
|
56
|
-
File::install(File.join(srcdir, f), File.join(destdir, f), mode, true)
|
57
|
-
else
|
58
|
-
from = File.join(srcdir, f)
|
59
|
-
to = File.join(destdir, f)
|
60
|
-
shebangify(from) do |sf|
|
61
|
-
$deferr.print from, " -> ", File::catname(from, to), "\n"
|
62
|
-
$deferr.printf "chmod %04o %s\n", mode, to
|
63
|
-
File::install(sf, to, mode, false)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
#}}}
|
68
|
-
end
|
69
|
-
def shebangify f
|
70
|
-
#{{{
|
71
|
-
open(f) do |fd|
|
72
|
-
buf = fd.read 42
|
73
|
-
if buf =~ %r/^\s*#\s*!.*ruby/o
|
74
|
-
ftmp = Tempfile::new("#{ $$ }_#{ File::basename(f) }")
|
75
|
-
begin
|
76
|
-
fd.rewind
|
77
|
-
ftmp.puts "#!#{ $ruby }"
|
78
|
-
while((buf = fd.read(8192)))
|
79
|
-
ftmp.write buf
|
80
|
-
end
|
81
|
-
ftmp.close
|
82
|
-
yield ftmp.path
|
83
|
-
ensure
|
84
|
-
ftmp.close!
|
85
|
-
end
|
86
|
-
else
|
87
|
-
yield f
|
88
|
-
end
|
89
|
-
end
|
90
|
-
#}}}
|
91
|
-
end
|
92
|
-
def ARGV.switch
|
93
|
-
#{{{
|
94
|
-
return nil if self.empty?
|
95
|
-
arg = self.shift
|
96
|
-
return nil if arg == '--'
|
97
|
-
if arg =~ /^-(.)(.*)/
|
98
|
-
return arg if $1 == '-'
|
99
|
-
raise 'unknown switch "-"' if $2.index('-')
|
100
|
-
self.unshift "-#{$2}" if $2.size > 0
|
101
|
-
"-#{$1}"
|
102
|
-
else
|
103
|
-
self.unshift arg
|
104
|
-
nil
|
105
|
-
end
|
106
|
-
#}}}
|
107
|
-
end
|
108
|
-
def ARGV.req_arg
|
109
|
-
#{{{
|
110
|
-
self.shift || raise('missing argument')
|
111
|
-
#}}}
|
112
|
-
end
|
113
|
-
def linkify d, linked = []
|
114
|
-
#--{{{
|
115
|
-
if test ?d, d
|
116
|
-
versioned = Dir[ File::join(d, "*-[0-9].[0-9].[0-9].rb") ]
|
117
|
-
versioned.each do |v|
|
118
|
-
src, dst = v, v.gsub(%r/\-[\d\.]+\.rb$/, '.rb')
|
119
|
-
lnk = nil
|
120
|
-
begin
|
121
|
-
if test ?l, dst
|
122
|
-
lnk = "#{ dst }.lnk"
|
123
|
-
puts "#{ dst } -> #{ lnk }"
|
124
|
-
File::rename dst, lnk
|
125
|
-
end
|
126
|
-
unless test ?e, dst
|
127
|
-
puts "#{ src } -> #{ dst }"
|
128
|
-
File::copy src, dst
|
129
|
-
linked << dst
|
130
|
-
end
|
131
|
-
ensure
|
132
|
-
if lnk
|
133
|
-
at_exit do
|
134
|
-
puts "#{ lnk } -> #{ dst }"
|
135
|
-
File::rename lnk, dst
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
linked
|
142
|
-
#--}}}
|
143
|
-
end
|
144
|
-
|
145
|
-
|
146
|
-
#
|
147
|
-
# main program
|
148
|
-
#
|
149
|
-
|
150
|
-
libdir = $site_libdir
|
151
|
-
bindir = $bindir
|
152
|
-
no_linkify = false
|
153
|
-
linked = nil
|
154
|
-
help = false
|
155
|
-
|
156
|
-
usage = <<-usage
|
157
|
-
#{ File::basename $0 }
|
158
|
-
-d, --destdir <destdir>
|
159
|
-
-l, --libdir <libdir>
|
160
|
-
-b, --bindir <bindir>
|
161
|
-
-r, --ruby <ruby>
|
162
|
-
-n, --no_linkify
|
163
|
-
-s, --sudo
|
164
|
-
-h, --help
|
165
|
-
usage
|
166
|
-
|
167
|
-
begin
|
168
|
-
while switch = ARGV.switch
|
169
|
-
case switch
|
170
|
-
when '-d', '--destdir'
|
171
|
-
libdir = ARGV.req_arg
|
172
|
-
when '-l', '--libdir'
|
173
|
-
libdir = ARGV.req_arg
|
174
|
-
when '-b', '--bindir'
|
175
|
-
bindir = ARGV.req_arg
|
176
|
-
when '-r', '--ruby'
|
177
|
-
$ruby = ARGV.req_arg
|
178
|
-
when '-n', '--no_linkify'
|
179
|
-
no_linkify = true
|
180
|
-
when '-s', '--sudo'
|
181
|
-
sudo = 'sudo'
|
182
|
-
when '-h', '--help'
|
183
|
-
help = true
|
184
|
-
else
|
185
|
-
raise "unknown switch #{switch.dump}"
|
186
|
-
end
|
187
|
-
end
|
188
|
-
rescue
|
189
|
-
STDERR.puts $!.to_s
|
190
|
-
STDERR.puts usage
|
191
|
-
exit 1
|
192
|
-
end
|
193
|
-
|
194
|
-
if help
|
195
|
-
STDOUT.puts usage
|
196
|
-
exit
|
197
|
-
end
|
198
|
-
|
199
|
-
system "#{ sudo } #{ $ruby } pre-install.rb" if test(?s, 'pre-install.rb')
|
200
|
-
|
201
|
-
unless no_linkify
|
202
|
-
linked = linkify('lib') + linkify('bin')
|
203
|
-
end
|
204
|
-
|
205
|
-
system "#{ $ruby } extconf.rb && make && #{ sudo } make install" if test(?s, 'extconf.rb')
|
206
|
-
|
207
|
-
install_rb(LIBDIR, libdir, LIBDIR_MODE)
|
208
|
-
install_rb(BINDIR, bindir, BINDIR_MODE, bin=true)
|
209
|
-
|
210
|
-
if linked
|
211
|
-
linked.each{|path| File::rm_f path}
|
212
|
-
end
|
213
|
-
|
214
|
-
system "#{ sudo } #{ $ruby } post-install.rb" if test(?s, 'post-install.rb')
|