forkoff 0.0.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +96 -84
- data/forkoff.gemspec +27 -0
- data/lib/forkoff.rb +17 -15
- data/rakefile +223 -0
- data/readme.erb +76 -0
- data/test/forkoff.rb +54 -0
- metadata +15 -14
- data/gemspec.rb +0 -35
- data/gen_readme.rb +0 -32
- data/install.rb +0 -214
data/README
CHANGED
@@ -9,6 +9,7 @@ SYNOPSIS
|
|
9
9
|
URI
|
10
10
|
|
11
11
|
http://rubyforge.org/projects/codeforpeople
|
12
|
+
http://github.com/ahoward/forkoff
|
12
13
|
|
13
14
|
INSTALL
|
14
15
|
|
@@ -20,48 +21,9 @@ DESCRIPTION
|
|
20
21
|
child process and collecting the results. forkoff can limit the number of
|
21
22
|
child processes which is, by default, 2.
|
22
23
|
|
23
|
-
HISTORY
|
24
|
-
0.0.4
|
25
|
-
- code re-org
|
26
|
-
- add :strategy option
|
27
|
-
- default number of processes is 2, not 8
|
28
|
-
|
29
|
-
0.0.1
|
30
|
-
|
31
|
-
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
32
|
-
channel. in this way the producers do not build up a massize parllel data
|
33
|
-
structure but provide data to the consumers only as fast as they can fork
|
34
|
-
and proccess it. basically for a 4 process run you'll end up with 4
|
35
|
-
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
36
|
-
thread popping of jobs, forking, and yielding results.
|
37
|
-
|
38
|
-
- removed use of Queue for capturing the output. now it's simply an array
|
39
|
-
of arrays which removed some sync overhead.
|
40
|
-
|
41
|
-
- you can configure the number of processes globally with
|
42
|
-
|
43
|
-
Forkoff.default['proccess'] = 4
|
44
|
-
|
45
|
-
- you can now pass either an options hash
|
46
|
-
|
47
|
-
forkoff( :processes => 2 ) ...
|
48
|
-
|
49
|
-
or plain vanilla number
|
50
|
-
|
51
|
-
forkoff( 2 ) ...
|
52
|
-
|
53
|
-
to the forkoff call
|
54
|
-
|
55
|
-
- default number of processes is 8, not 2
|
56
|
-
|
57
|
-
|
58
|
-
0.0.0
|
59
|
-
|
60
|
-
initial version
|
61
|
-
|
62
|
-
|
63
24
|
SAMPLES
|
64
25
|
|
26
|
+
|
65
27
|
<========< samples/a.rb >========>
|
66
28
|
|
67
29
|
~ > cat samples/a.rb
|
@@ -76,8 +38,8 @@ SAMPLES
|
|
76
38
|
|
77
39
|
~ > ruby samples/a.rb
|
78
40
|
|
79
|
-
hey from
|
80
|
-
you from
|
41
|
+
hey from 7907
|
42
|
+
you from 7908
|
81
43
|
|
82
44
|
|
83
45
|
<========< samples/b.rb >========>
|
@@ -107,7 +69,7 @@ SAMPLES
|
|
107
69
|
|
108
70
|
~ > ruby samples/b.rb
|
109
71
|
|
110
|
-
elapsed: 4.
|
72
|
+
elapsed: 4.19184589385986
|
111
73
|
results: [0, 1, 4, 9, 16, 25, 36, 49]
|
112
74
|
|
113
75
|
|
@@ -151,71 +113,70 @@ SAMPLES
|
|
151
113
|
~ > ruby samples/c.rb
|
152
114
|
|
153
115
|
|
154
|
-
pid:
|
155
|
-
elapsed: 3.
|
116
|
+
pid: 7922
|
117
|
+
elapsed: 3.37899208068848
|
156
118
|
|
157
119
|
---
|
158
120
|
a: |
|
159
|
-
-+-
|
160
|
-
|-+-
|
161
|
-
|-+-
|
162
|
-
\-+-
|
121
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
122
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
123
|
+
|-+- 07924 ahoward (ruby)
|
124
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
163
125
|
|
164
126
|
---
|
165
127
|
b: |
|
166
|
-
-+-
|
167
|
-
|-+-
|
168
|
-
|-+-
|
169
|
-
\-+-
|
128
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
129
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
130
|
+
|-+- 07924 ahoward ruby -Ilib samples/c.rb
|
131
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
170
132
|
|
171
133
|
---
|
172
134
|
c: |
|
173
|
-
-+-
|
174
|
-
|-+-
|
175
|
-
|-+-
|
176
|
-
\-+-
|
135
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
136
|
+
|-+- 07923 ahoward ruby -Ilib samples/c.rb
|
137
|
+
|-+- 07924 ahoward (ruby)
|
138
|
+
\-+- 07925 ahoward ruby -Ilib samples/c.rb
|
177
139
|
|
178
140
|
---
|
179
141
|
d: |
|
180
|
-
-+-
|
181
|
-
|-+-
|
182
|
-
|
183
|
-
|
142
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
143
|
+
|-+- 07932 ahoward ruby -Ilib samples/c.rb
|
144
|
+
|--- 07933 ahoward ruby -Ilib samples/c.rb
|
145
|
+
\--- 07934 ahoward ruby -Ilib samples/c.rb
|
184
146
|
|
185
147
|
---
|
186
148
|
e: |
|
187
|
-
-+-
|
188
|
-
|
189
|
-
|-+-
|
190
|
-
\-+-
|
149
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
150
|
+
|--- 07932 ahoward (ruby)
|
151
|
+
|-+- 07933 ahoward ruby -Ilib samples/c.rb
|
152
|
+
\-+- 07934 ahoward (ruby)
|
191
153
|
|
192
154
|
---
|
193
155
|
f: |
|
194
|
-
-+-
|
195
|
-
|
196
|
-
|-+-
|
197
|
-
\-+-
|
156
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
157
|
+
|--- 07932 ahoward (ruby)
|
158
|
+
|-+- 07933 ahoward ruby -Ilib samples/c.rb
|
159
|
+
\-+- 07934 ahoward ruby -Ilib samples/c.rb
|
198
160
|
|
199
161
|
---
|
200
162
|
g: |
|
201
|
-
-+-
|
202
|
-
|-+-
|
203
|
-
|
204
|
-
|
163
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
164
|
+
|-+- 07941 ahoward ruby -Ilib samples/c.rb
|
165
|
+
|--- 07942 ahoward ruby -Ilib samples/c.rb
|
166
|
+
\--- 07943 ahoward ruby -Ilib samples/c.rb
|
205
167
|
|
206
168
|
---
|
207
169
|
h: |
|
208
|
-
-+-
|
209
|
-
|-+-
|
210
|
-
|-+-
|
211
|
-
|
170
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
171
|
+
|-+- 07941 ahoward (ruby)
|
172
|
+
|-+- 07942 ahoward ruby -Ilib samples/c.rb
|
173
|
+
\--- 07943 ahoward ruby -Ilib samples/c.rb
|
212
174
|
|
213
175
|
---
|
214
176
|
i: |
|
215
|
-
-+-
|
216
|
-
|
217
|
-
|
218
|
-
\-+- 01092 ahoward ruby -Ilib samples/c.rb
|
177
|
+
-+- 07922 ahoward ruby -Ilib samples/c.rb
|
178
|
+
|--- 07942 ahoward (ruby)
|
179
|
+
\-+- 07943 ahoward ruby -Ilib samples/c.rb
|
219
180
|
|
220
181
|
|
221
182
|
|
@@ -236,7 +197,58 @@ SAMPLES
|
|
236
197
|
|
237
198
|
~ > ruby samples/d.rb
|
238
199
|
|
239
|
-
hey from
|
240
|
-
you from
|
241
|
-
guys from
|
200
|
+
hey from 7953
|
201
|
+
you from 7954
|
202
|
+
guys from 7955
|
203
|
+
|
242
204
|
|
205
|
+
|
206
|
+
HISTORY
|
207
|
+
1.1.0
|
208
|
+
- move to a model with one work queue and signals sent from consumers to
|
209
|
+
producer to noitify ready state. this let's smaller jobs race through a
|
210
|
+
single process even while a larger job may have one sub-process bound up.
|
211
|
+
incorporates a fix from http://github.com/fredrikj/forkoff which meant
|
212
|
+
some processes would lag behind when jobs didn't have similar execution
|
213
|
+
times.
|
214
|
+
|
215
|
+
1.0.0
|
216
|
+
- move to github
|
217
|
+
|
218
|
+
0.0.4
|
219
|
+
- code re-org
|
220
|
+
- add :strategy option
|
221
|
+
- default number of processes is 2, not 8
|
222
|
+
|
223
|
+
0.0.1
|
224
|
+
|
225
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
226
|
+
channel. in this way the producers do not build up a massize parllel data
|
227
|
+
structure but provide data to the consumers only as fast as they can fork
|
228
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
229
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
230
|
+
thread popping of jobs, forking, and yielding results.
|
231
|
+
|
232
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
233
|
+
of arrays which removed some sync overhead.
|
234
|
+
|
235
|
+
- you can configure the number of processes globally with
|
236
|
+
|
237
|
+
Forkoff.default['proccess'] = 4
|
238
|
+
|
239
|
+
- you can now pass either an options hash
|
240
|
+
|
241
|
+
forkoff( :processes => 2 ) ...
|
242
|
+
|
243
|
+
or plain vanilla number
|
244
|
+
|
245
|
+
forkoff( 2 ) ...
|
246
|
+
|
247
|
+
to the forkoff call
|
248
|
+
|
249
|
+
- default number of processes is 8, not 2
|
250
|
+
|
251
|
+
|
252
|
+
0.0.0
|
253
|
+
|
254
|
+
initial version
|
data/forkoff.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
## forkoff.gemspec
|
2
|
+
#
|
3
|
+
|
4
|
+
Gem::Specification::new do |spec|
|
5
|
+
spec.name = "forkoff"
|
6
|
+
spec.description = 'brain-dead simple parallel processing for ruby'
|
7
|
+
spec.version = "1.1.0"
|
8
|
+
spec.platform = Gem::Platform::RUBY
|
9
|
+
spec.summary = "forkoff"
|
10
|
+
|
11
|
+
spec.files = ["forkoff.gemspec", "lib", "lib/forkoff.rb", "rakefile", "README", "readme.erb", "samples", "samples/a.rb", "samples/b.rb", "samples/c.rb", "samples/d.rb", "test", "test/forkoff.rb"]
|
12
|
+
spec.executables = []
|
13
|
+
|
14
|
+
spec.require_path = "lib"
|
15
|
+
|
16
|
+
spec.has_rdoc = true
|
17
|
+
spec.test_files = "test/forkoff.rb"
|
18
|
+
#spec.add_dependency 'lib', '>= version'
|
19
|
+
#spec.add_dependency 'fattr'
|
20
|
+
|
21
|
+
spec.extensions.push(*[])
|
22
|
+
|
23
|
+
spec.rubyforge_project = "codeforpeople"
|
24
|
+
spec.author = "Ara T. Howard"
|
25
|
+
spec.email = "ara.t.howard@gmail.com"
|
26
|
+
spec.homepage = "http://github.com/ahoward/forkoff/tree/master"
|
27
|
+
end
|
data/lib/forkoff.rb
CHANGED
@@ -2,13 +2,17 @@ require 'thread'
|
|
2
2
|
|
3
3
|
module Forkoff
|
4
4
|
def version
|
5
|
-
'
|
5
|
+
'1.1.0'
|
6
6
|
end
|
7
7
|
|
8
8
|
def done
|
9
9
|
@done ||= Object.new
|
10
10
|
end
|
11
11
|
|
12
|
+
def ready
|
13
|
+
@ready ||= Object.new
|
14
|
+
end
|
15
|
+
|
12
16
|
def default
|
13
17
|
@default ||= { 'processes' => 2 }
|
14
18
|
end
|
@@ -137,7 +141,8 @@ module Enumerable
|
|
137
141
|
options = { 'processes' => Integer(options) } unless Hash === options
|
138
142
|
n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
|
139
143
|
strategy = options['strategy'] || options[:strategy] || 'pipe'
|
140
|
-
|
144
|
+
p2c = Queue.new
|
145
|
+
c2p = Queue.new
|
141
146
|
results = Array.new(n){ [] }
|
142
147
|
|
143
148
|
#
|
@@ -151,7 +156,8 @@ module Enumerable
|
|
151
156
|
Thread.current.abort_on_exception = true
|
152
157
|
|
153
158
|
loop do
|
154
|
-
|
159
|
+
c2p.push( Forkoff.ready )
|
160
|
+
value = p2c.pop
|
155
161
|
break if value == Forkoff.done
|
156
162
|
args, index = value
|
157
163
|
|
@@ -177,22 +183,18 @@ module Enumerable
|
|
177
183
|
#
|
178
184
|
# producers
|
179
185
|
#
|
180
|
-
|
181
|
-
|
182
|
-
n.times do |i|
|
183
|
-
thread = Thread.new do
|
186
|
+
producer =
|
187
|
+
Thread.new do
|
184
188
|
Thread.current.abort_on_exception = true
|
185
|
-
each_with_index do |args,
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
+
each_with_index do |args, i|
|
190
|
+
ready = c2p.pop
|
191
|
+
p2c.push( [args, i] )
|
192
|
+
end
|
193
|
+
n.times do |i|
|
194
|
+
p2c.push( Forkoff.done )
|
189
195
|
end
|
190
|
-
qs[ i ].push( Forkoff.done )
|
191
196
|
end
|
192
197
|
|
193
|
-
producers << thread
|
194
|
-
end
|
195
|
-
|
196
198
|
#
|
197
199
|
# wait for all consumers to complete
|
198
200
|
#
|
data/rakefile
ADDED
@@ -0,0 +1,223 @@
|
|
1
|
+
|
2
|
+
This.rubyforge_project = 'codeforpeople'
|
3
|
+
This.author = "Ara T. Howard"
|
4
|
+
This.email = "ara.t.howard@gmail.com"
|
5
|
+
This.homepage = "http://github.com/ahoward/#{ This.lib }/tree/master"
|
6
|
+
|
7
|
+
|
8
|
+
task :default do
|
9
|
+
puts(Rake::Task.tasks.map{|task| task.name} - ['default'])
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
task :gemspec do
|
14
|
+
ignore_extensions = 'git', 'svn', 'tmp', /sw./, 'bak', 'gem'
|
15
|
+
ignore_directories = 'pkg'
|
16
|
+
|
17
|
+
shiteless =
|
18
|
+
lambda do |list|
|
19
|
+
list.delete_if do |entry|
|
20
|
+
next unless test(?e, entry)
|
21
|
+
extension = File.basename(entry).split(%r/[.]/).last
|
22
|
+
ignore_extensions.any?{|ext| ext === extension}
|
23
|
+
end
|
24
|
+
list.delete_if do |entry|
|
25
|
+
next unless test(?d, entry)
|
26
|
+
dirname = File.expand_path(entry)
|
27
|
+
ignore_directories.any?{|dir| File.expand_path(dir) == dirname}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
lib = This.lib
|
32
|
+
version = This.version
|
33
|
+
files = shiteless[Dir::glob("**/**")]
|
34
|
+
executables = shiteless[Dir::glob("bin/*")].map{|exe| File.basename(exe)}
|
35
|
+
has_rdoc = true #File.exist?('doc')
|
36
|
+
test_files = "test/#{ lib }.rb" if File.file?("test/#{ lib }.rb")
|
37
|
+
|
38
|
+
extensions = This.extensions
|
39
|
+
if extensions.nil?
|
40
|
+
%w( Makefile configure extconf.rb ).each do |ext|
|
41
|
+
extensions << ext if File.exists?(ext)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
extensions = [extensions].flatten.compact
|
45
|
+
|
46
|
+
template =
|
47
|
+
if test(?e, 'gemspec.erb')
|
48
|
+
Template{ IO.read('gemspec.erb') }
|
49
|
+
else
|
50
|
+
Template {
|
51
|
+
<<-__
|
52
|
+
## #{ lib }.gemspec
|
53
|
+
#
|
54
|
+
|
55
|
+
Gem::Specification::new do |spec|
|
56
|
+
spec.name = #{ lib.inspect }
|
57
|
+
spec.description = 'brain-dead simple parallel processing for ruby'
|
58
|
+
spec.version = #{ version.inspect }
|
59
|
+
spec.platform = Gem::Platform::RUBY
|
60
|
+
spec.summary = #{ lib.inspect }
|
61
|
+
|
62
|
+
spec.files = #{ files.inspect }
|
63
|
+
spec.executables = #{ executables.inspect }
|
64
|
+
|
65
|
+
spec.require_path = "lib"
|
66
|
+
|
67
|
+
spec.has_rdoc = #{ has_rdoc.inspect }
|
68
|
+
spec.test_files = #{ test_files.inspect }
|
69
|
+
#spec.add_dependency 'lib', '>= version'
|
70
|
+
#spec.add_dependency 'fattr'
|
71
|
+
|
72
|
+
spec.extensions.push(*#{ extensions.inspect })
|
73
|
+
|
74
|
+
spec.rubyforge_project = #{ This.rubyforge_project.inspect }
|
75
|
+
spec.author = #{ This.author.inspect }
|
76
|
+
spec.email = #{ This.email.inspect }
|
77
|
+
spec.homepage = #{ This.homepage.inspect }
|
78
|
+
end
|
79
|
+
__
|
80
|
+
}
|
81
|
+
end
|
82
|
+
|
83
|
+
open("#{ lib }.gemspec", "w"){|fd| fd.puts template}
|
84
|
+
This.gemspec = "#{ lib }.gemspec"
|
85
|
+
end
|
86
|
+
|
87
|
+
task :gem => [:clean, :gemspec] do
|
88
|
+
Fu.mkdir_p This.pkgdir
|
89
|
+
before = Dir['*.gem']
|
90
|
+
cmd = "gem build #{ This.gemspec }"
|
91
|
+
`#{ cmd }`
|
92
|
+
after = Dir['*.gem']
|
93
|
+
gem = ((after - before).first || after.first) or abort('no gem!')
|
94
|
+
Fu.mv gem, This.pkgdir
|
95
|
+
This.gem = File.basename(gem)
|
96
|
+
end
|
97
|
+
|
98
|
+
task :readme do
|
99
|
+
samples = ''
|
100
|
+
prompt = '~ > '
|
101
|
+
lib = This.lib
|
102
|
+
version = This.version
|
103
|
+
|
104
|
+
Dir['sample*/*'].sort.each do |sample|
|
105
|
+
samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
|
106
|
+
|
107
|
+
cmd = "cat #{ sample }"
|
108
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
109
|
+
samples << Util.indent(`#{ cmd }`, 4) << "\n"
|
110
|
+
|
111
|
+
cmd = "ruby #{ sample }"
|
112
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
113
|
+
|
114
|
+
cmd = "ruby -e'STDOUT.sync=true; exec %(ruby -Ilib #{ sample })'"
|
115
|
+
samples << Util.indent(`#{ cmd } 2>&1`, 4) << "\n"
|
116
|
+
end
|
117
|
+
|
118
|
+
template =
|
119
|
+
if test(?e, 'readme.erb')
|
120
|
+
Template{ IO.read('readme.erb') }
|
121
|
+
else
|
122
|
+
Template {
|
123
|
+
<<-__
|
124
|
+
NAME
|
125
|
+
#{ lib }
|
126
|
+
|
127
|
+
DESCRIPTION
|
128
|
+
|
129
|
+
INSTALL
|
130
|
+
gem install #{ lib }
|
131
|
+
|
132
|
+
SAMPLES
|
133
|
+
#{ samples }
|
134
|
+
__
|
135
|
+
}
|
136
|
+
end
|
137
|
+
|
138
|
+
open("README", "w"){|fd| fd.puts template}
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
task :clean do
|
143
|
+
Dir[File.join(This.pkgdir, '**/**')].each{|entry| Fu.rm_rf(entry)}
|
144
|
+
end
|
145
|
+
|
146
|
+
|
147
|
+
task :release => [:clean, :gemspec, :gem] do
|
148
|
+
gems = Dir[File.join(This.pkgdir, '*.gem')].flatten
|
149
|
+
raise "which one? : #{ gems.inspect }" if gems.size > 1
|
150
|
+
raise "no gems?" if gems.size < 1
|
151
|
+
cmd = "rubyforge login && rubyforge add_release #{ This.rubyforge_project } #{ This.lib } #{ This.version } #{ This.pkgdir }/#{ This.gem }"
|
152
|
+
puts cmd
|
153
|
+
system cmd
|
154
|
+
end
|
155
|
+
|
156
|
+
|
157
|
+
|
158
|
+
|
159
|
+
|
160
|
+
BEGIN {
|
161
|
+
$VERBOSE = nil
|
162
|
+
|
163
|
+
require 'ostruct'
|
164
|
+
require 'erb'
|
165
|
+
require 'fileutils'
|
166
|
+
|
167
|
+
Fu = FileUtils
|
168
|
+
|
169
|
+
This = OpenStruct.new
|
170
|
+
|
171
|
+
This.file = File.expand_path(__FILE__)
|
172
|
+
This.dir = File.dirname(This.file)
|
173
|
+
This.pkgdir = File.join(This.dir, 'pkg')
|
174
|
+
|
175
|
+
lib = ENV['LIB']
|
176
|
+
unless lib
|
177
|
+
lib = File.basename(Dir.pwd)
|
178
|
+
end
|
179
|
+
This.lib = lib
|
180
|
+
|
181
|
+
version = ENV['VERSION']
|
182
|
+
unless version
|
183
|
+
name = lib.capitalize
|
184
|
+
require "./lib/#{ lib }"
|
185
|
+
version = eval(name).send(:version)
|
186
|
+
end
|
187
|
+
This.version = version
|
188
|
+
|
189
|
+
abort('no lib') unless This.lib
|
190
|
+
abort('no version') unless This.version
|
191
|
+
|
192
|
+
module Util
|
193
|
+
def indent(s, n = 2)
|
194
|
+
s = unindent(s)
|
195
|
+
ws = ' ' * n
|
196
|
+
s.gsub(%r/^/, ws)
|
197
|
+
end
|
198
|
+
|
199
|
+
def unindent(s)
|
200
|
+
indent = nil
|
201
|
+
s.each do |line|
|
202
|
+
next if line =~ %r/^\s*$/
|
203
|
+
indent = line[%r/^\s*/] and break
|
204
|
+
end
|
205
|
+
indent ? s.gsub(%r/^#{ indent }/, "") : s
|
206
|
+
end
|
207
|
+
extend self
|
208
|
+
end
|
209
|
+
|
210
|
+
class Template
|
211
|
+
def initialize(&block)
|
212
|
+
@block = block
|
213
|
+
@template = block.call.to_s
|
214
|
+
end
|
215
|
+
def expand(b=nil)
|
216
|
+
ERB.new(Util.unindent(@template)).result(b||@block)
|
217
|
+
end
|
218
|
+
alias_method 'to_s', 'expand'
|
219
|
+
end
|
220
|
+
def Template(*args, &block) Template.new(*args, &block) end
|
221
|
+
|
222
|
+
Dir.chdir(This.dir)
|
223
|
+
}
|
data/readme.erb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
NAME
|
2
|
+
|
3
|
+
forkoff
|
4
|
+
|
5
|
+
SYNOPSIS
|
6
|
+
|
7
|
+
brain-dead simple parallel processing for ruby
|
8
|
+
|
9
|
+
URI
|
10
|
+
|
11
|
+
http://rubyforge.org/projects/codeforpeople
|
12
|
+
http://github.com/ahoward/forkoff
|
13
|
+
|
14
|
+
INSTALL
|
15
|
+
|
16
|
+
gem install forkoff
|
17
|
+
|
18
|
+
DESCRIPTION
|
19
|
+
|
20
|
+
forkoff works for any enumerable object, iterating a code block to run in a
|
21
|
+
child process and collecting the results. forkoff can limit the number of
|
22
|
+
child processes which is, by default, 2.
|
23
|
+
|
24
|
+
SAMPLES
|
25
|
+
|
26
|
+
<%= samples %>
|
27
|
+
|
28
|
+
HISTORY
|
29
|
+
1.1.0
|
30
|
+
- move to a model with one work queue and signals sent from consumers to
|
31
|
+
producer to noitify ready state. this let's smaller jobs race through a
|
32
|
+
single process even while a larger job may have one sub-process bound up.
|
33
|
+
incorporates a fix from http://github.com/fredrikj/forkoff which meant
|
34
|
+
some processes would lag behind when jobs didn't have similar execution
|
35
|
+
times.
|
36
|
+
|
37
|
+
1.0.0
|
38
|
+
- move to github
|
39
|
+
|
40
|
+
0.0.4
|
41
|
+
- code re-org
|
42
|
+
- add :strategy option
|
43
|
+
- default number of processes is 2, not 8
|
44
|
+
|
45
|
+
0.0.1
|
46
|
+
|
47
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
48
|
+
channel. in this way the producers do not build up a massize parllel data
|
49
|
+
structure but provide data to the consumers only as fast as they can fork
|
50
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
51
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
52
|
+
thread popping of jobs, forking, and yielding results.
|
53
|
+
|
54
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
55
|
+
of arrays which removed some sync overhead.
|
56
|
+
|
57
|
+
- you can configure the number of processes globally with
|
58
|
+
|
59
|
+
Forkoff.default['proccess'] = 4
|
60
|
+
|
61
|
+
- you can now pass either an options hash
|
62
|
+
|
63
|
+
forkoff( :processes => 2 ) ...
|
64
|
+
|
65
|
+
or plain vanilla number
|
66
|
+
|
67
|
+
forkoff( 2 ) ...
|
68
|
+
|
69
|
+
to the forkoff call
|
70
|
+
|
71
|
+
- default number of processes is 8, not 2
|
72
|
+
|
73
|
+
|
74
|
+
0.0.0
|
75
|
+
|
76
|
+
initial version
|
data/test/forkoff.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
$:.unshift('.')
|
2
|
+
$:.unshift('./lib')
|
3
|
+
$:.unshift('..')
|
4
|
+
$:.unshift('../lib')
|
5
|
+
|
6
|
+
require('test/unit')
|
7
|
+
require('forkoff')
|
8
|
+
|
9
|
+
|
10
|
+
class T < Test::Unit::TestCase
|
11
|
+
|
12
|
+
# simple usage
|
13
|
+
#
|
14
|
+
|
15
|
+
def test_0010
|
16
|
+
results = [0,1,2].forkoff!{|n| [n, Process.pid]}
|
17
|
+
assert_equal(results, results.uniq, 'all ran in a different process')
|
18
|
+
end
|
19
|
+
|
20
|
+
# it's faster
|
21
|
+
#
|
22
|
+
def test_0020
|
23
|
+
n = 4
|
24
|
+
strategies = :forkoff, :each
|
25
|
+
|
26
|
+
4.times do
|
27
|
+
result = {}
|
28
|
+
|
29
|
+
strategies.each do |strategy|
|
30
|
+
a = Time.now.to_f
|
31
|
+
(0..4).send(strategy){|i| sleep 0.2}
|
32
|
+
b = Time.now.to_f
|
33
|
+
elapsed = b - a
|
34
|
+
result[strategy] = elapsed
|
35
|
+
end
|
36
|
+
|
37
|
+
assert result[:forkoff] < result[:each], 'forkoff is faster than each for long running tasks'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# in case of different execution times for different processes
|
42
|
+
#
|
43
|
+
def test_0030
|
44
|
+
a = Time.now.to_f
|
45
|
+
(0...4).forkoff(2) do |i|
|
46
|
+
sleep i.modulo(2)
|
47
|
+
end
|
48
|
+
b = Time.now.to_f
|
49
|
+
elapsed = b - a
|
50
|
+
assert elapsed < 2
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forkoff
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ara T. Howard
|
@@ -9,11 +9,11 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2009-10-11 00:00:00 -06:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
16
|
-
description:
|
16
|
+
description: brain-dead simple parallel processing for ruby
|
17
17
|
email: ara.t.howard@gmail.com
|
18
18
|
executables: []
|
19
19
|
|
@@ -22,19 +22,20 @@ extensions: []
|
|
22
22
|
extra_rdoc_files: []
|
23
23
|
|
24
24
|
files:
|
25
|
-
- gemspec
|
26
|
-
- gen_readme.rb
|
27
|
-
- install.rb
|
28
|
-
- lib
|
25
|
+
- forkoff.gemspec
|
29
26
|
- lib/forkoff.rb
|
27
|
+
- rakefile
|
30
28
|
- README
|
31
|
-
-
|
29
|
+
- readme.erb
|
32
30
|
- samples/a.rb
|
33
31
|
- samples/b.rb
|
34
32
|
- samples/c.rb
|
35
33
|
- samples/d.rb
|
36
|
-
|
37
|
-
|
34
|
+
- test/forkoff.rb
|
35
|
+
has_rdoc: true
|
36
|
+
homepage: http://github.com/ahoward/forkoff/tree/master
|
37
|
+
licenses: []
|
38
|
+
|
38
39
|
post_install_message:
|
39
40
|
rdoc_options: []
|
40
41
|
|
@@ -55,9 +56,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
55
56
|
requirements: []
|
56
57
|
|
57
58
|
rubyforge_project: codeforpeople
|
58
|
-
rubygems_version: 1.
|
59
|
+
rubygems_version: 1.3.5
|
59
60
|
signing_key:
|
60
|
-
specification_version:
|
61
|
+
specification_version: 3
|
61
62
|
summary: forkoff
|
62
|
-
test_files:
|
63
|
-
|
63
|
+
test_files:
|
64
|
+
- test/forkoff.rb
|
data/gemspec.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
lib, version = File::basename(File::dirname(File::expand_path(__FILE__))).split %r/-/, 2
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
|
5
|
-
Gem::Specification::new do |spec|
|
6
|
-
$VERBOSE = nil
|
7
|
-
|
8
|
-
shiteless = lambda do |list|
|
9
|
-
list.delete_if do |file|
|
10
|
-
file =~ %r/\.svn/ or
|
11
|
-
file =~ %r/\.tmp/
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
spec.name = lib
|
16
|
-
spec.version = version
|
17
|
-
spec.platform = Gem::Platform::RUBY
|
18
|
-
spec.summary = lib
|
19
|
-
|
20
|
-
spec.files = shiteless[Dir::glob("**/**")]
|
21
|
-
spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
|
22
|
-
|
23
|
-
spec.require_path = "lib"
|
24
|
-
|
25
|
-
spec.has_rdoc = File::exist? "doc"
|
26
|
-
spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
|
27
|
-
#spec.add_dependency 'lib', '>= version'
|
28
|
-
|
29
|
-
spec.extensions << "extconf.rb" if File::exists? "extconf.rb"
|
30
|
-
|
31
|
-
spec.author = "Ara T. Howard"
|
32
|
-
spec.email = "ara.t.howard@gmail.com"
|
33
|
-
spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
|
34
|
-
spec.rubyforge_project = 'codeforpeople'
|
35
|
-
end
|
data/gen_readme.rb
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
require 'pathname'
|
2
|
-
|
3
|
-
$VERBOSE=nil
|
4
|
-
|
5
|
-
def indent s, n = 2
|
6
|
-
ws = ' ' * n
|
7
|
-
s.gsub %r/^/, ws
|
8
|
-
end
|
9
|
-
|
10
|
-
template = IO::read 'README.tmpl'
|
11
|
-
|
12
|
-
samples = ''
|
13
|
-
prompt = '~ > '
|
14
|
-
|
15
|
-
Dir['sample*/*'].sort.each do |sample|
|
16
|
-
samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
|
17
|
-
|
18
|
-
cmd = "cat #{ sample }"
|
19
|
-
samples << indent(prompt + cmd, 2) << "\n\n"
|
20
|
-
samples << indent(`#{ cmd }`, 4) << "\n"
|
21
|
-
|
22
|
-
cmd = "ruby #{ sample }"
|
23
|
-
samples << indent(prompt + cmd, 2) << "\n\n"
|
24
|
-
|
25
|
-
cmd = "ruby -Ilib #{ sample }"
|
26
|
-
samples << indent(`#{ cmd } 2>&1`, 4) << "\n"
|
27
|
-
end
|
28
|
-
|
29
|
-
#samples.gsub! %r/^/, ' '
|
30
|
-
|
31
|
-
readme = template.gsub %r/^\s*@samples\s*$/, samples
|
32
|
-
print readme
|
data/install.rb
DELETED
@@ -1,214 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
require 'rbconfig'
|
3
|
-
require 'find'
|
4
|
-
require 'ftools'
|
5
|
-
require 'tempfile'
|
6
|
-
include Config
|
7
|
-
|
8
|
-
LIBDIR = "lib"
|
9
|
-
LIBDIR_MODE = 0644
|
10
|
-
|
11
|
-
BINDIR = "bin"
|
12
|
-
BINDIR_MODE = 0755
|
13
|
-
|
14
|
-
|
15
|
-
$srcdir = CONFIG["srcdir"]
|
16
|
-
$version = CONFIG["MAJOR"]+"."+CONFIG["MINOR"]
|
17
|
-
$libdir = File.join(CONFIG["libdir"], "ruby", $version)
|
18
|
-
$archdir = File.join($libdir, CONFIG["arch"])
|
19
|
-
$site_libdir = $:.find {|x| x =~ /site_ruby$/}
|
20
|
-
$bindir = CONFIG["bindir"] || CONFIG['BINDIR']
|
21
|
-
$ruby_install_name = CONFIG['ruby_install_name'] || CONFIG['RUBY_INSTALL_NAME'] || 'ruby'
|
22
|
-
$ruby_ext = CONFIG['EXEEXT'] || ''
|
23
|
-
$ruby = File.join($bindir, ($ruby_install_name + $ruby_ext))
|
24
|
-
|
25
|
-
if !$site_libdir
|
26
|
-
$site_libdir = File.join($libdir, "site_ruby")
|
27
|
-
elsif $site_libdir !~ %r/#{Regexp.quote($version)}/
|
28
|
-
$site_libdir = File.join($site_libdir, $version)
|
29
|
-
end
|
30
|
-
|
31
|
-
def install_rb(srcdir=nil, destdir=nil, mode=nil, bin=nil)
|
32
|
-
#{{{
|
33
|
-
path = []
|
34
|
-
dir = []
|
35
|
-
Find.find(srcdir) do |f|
|
36
|
-
next unless FileTest.file?(f)
|
37
|
-
next if (f = f[srcdir.length+1..-1]) == nil
|
38
|
-
next if (/CVS$/ =~ File.dirname(f))
|
39
|
-
next if (/\.svn/ =~ File.dirname(f))
|
40
|
-
next if f =~ %r/\.lnk/
|
41
|
-
next if f =~ %r/\.svn/
|
42
|
-
next if f =~ %r/\.swp/
|
43
|
-
next if f =~ %r/\.svn/
|
44
|
-
path.push f
|
45
|
-
dir |= [File.dirname(f)]
|
46
|
-
end
|
47
|
-
for f in dir
|
48
|
-
next if f == "."
|
49
|
-
next if f == "CVS"
|
50
|
-
File::makedirs(File.join(destdir, f))
|
51
|
-
end
|
52
|
-
for f in path
|
53
|
-
next if (/\~$/ =~ f)
|
54
|
-
next if (/^\./ =~ File.basename(f))
|
55
|
-
unless bin
|
56
|
-
File::install(File.join(srcdir, f), File.join(destdir, f), mode, true)
|
57
|
-
else
|
58
|
-
from = File.join(srcdir, f)
|
59
|
-
to = File.join(destdir, f)
|
60
|
-
shebangify(from) do |sf|
|
61
|
-
$deferr.print from, " -> ", File::catname(from, to), "\n"
|
62
|
-
$deferr.printf "chmod %04o %s\n", mode, to
|
63
|
-
File::install(sf, to, mode, false)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
#}}}
|
68
|
-
end
|
69
|
-
def shebangify f
|
70
|
-
#{{{
|
71
|
-
open(f) do |fd|
|
72
|
-
buf = fd.read 42
|
73
|
-
if buf =~ %r/^\s*#\s*!.*ruby/o
|
74
|
-
ftmp = Tempfile::new("#{ $$ }_#{ File::basename(f) }")
|
75
|
-
begin
|
76
|
-
fd.rewind
|
77
|
-
ftmp.puts "#!#{ $ruby }"
|
78
|
-
while((buf = fd.read(8192)))
|
79
|
-
ftmp.write buf
|
80
|
-
end
|
81
|
-
ftmp.close
|
82
|
-
yield ftmp.path
|
83
|
-
ensure
|
84
|
-
ftmp.close!
|
85
|
-
end
|
86
|
-
else
|
87
|
-
yield f
|
88
|
-
end
|
89
|
-
end
|
90
|
-
#}}}
|
91
|
-
end
|
92
|
-
def ARGV.switch
|
93
|
-
#{{{
|
94
|
-
return nil if self.empty?
|
95
|
-
arg = self.shift
|
96
|
-
return nil if arg == '--'
|
97
|
-
if arg =~ /^-(.)(.*)/
|
98
|
-
return arg if $1 == '-'
|
99
|
-
raise 'unknown switch "-"' if $2.index('-')
|
100
|
-
self.unshift "-#{$2}" if $2.size > 0
|
101
|
-
"-#{$1}"
|
102
|
-
else
|
103
|
-
self.unshift arg
|
104
|
-
nil
|
105
|
-
end
|
106
|
-
#}}}
|
107
|
-
end
|
108
|
-
def ARGV.req_arg
|
109
|
-
#{{{
|
110
|
-
self.shift || raise('missing argument')
|
111
|
-
#}}}
|
112
|
-
end
|
113
|
-
def linkify d, linked = []
|
114
|
-
#--{{{
|
115
|
-
if test ?d, d
|
116
|
-
versioned = Dir[ File::join(d, "*-[0-9].[0-9].[0-9].rb") ]
|
117
|
-
versioned.each do |v|
|
118
|
-
src, dst = v, v.gsub(%r/\-[\d\.]+\.rb$/, '.rb')
|
119
|
-
lnk = nil
|
120
|
-
begin
|
121
|
-
if test ?l, dst
|
122
|
-
lnk = "#{ dst }.lnk"
|
123
|
-
puts "#{ dst } -> #{ lnk }"
|
124
|
-
File::rename dst, lnk
|
125
|
-
end
|
126
|
-
unless test ?e, dst
|
127
|
-
puts "#{ src } -> #{ dst }"
|
128
|
-
File::copy src, dst
|
129
|
-
linked << dst
|
130
|
-
end
|
131
|
-
ensure
|
132
|
-
if lnk
|
133
|
-
at_exit do
|
134
|
-
puts "#{ lnk } -> #{ dst }"
|
135
|
-
File::rename lnk, dst
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
linked
|
142
|
-
#--}}}
|
143
|
-
end
|
144
|
-
|
145
|
-
|
146
|
-
#
|
147
|
-
# main program
|
148
|
-
#
|
149
|
-
|
150
|
-
libdir = $site_libdir
|
151
|
-
bindir = $bindir
|
152
|
-
no_linkify = false
|
153
|
-
linked = nil
|
154
|
-
help = false
|
155
|
-
|
156
|
-
usage = <<-usage
|
157
|
-
#{ File::basename $0 }
|
158
|
-
-d, --destdir <destdir>
|
159
|
-
-l, --libdir <libdir>
|
160
|
-
-b, --bindir <bindir>
|
161
|
-
-r, --ruby <ruby>
|
162
|
-
-n, --no_linkify
|
163
|
-
-s, --sudo
|
164
|
-
-h, --help
|
165
|
-
usage
|
166
|
-
|
167
|
-
begin
|
168
|
-
while switch = ARGV.switch
|
169
|
-
case switch
|
170
|
-
when '-d', '--destdir'
|
171
|
-
libdir = ARGV.req_arg
|
172
|
-
when '-l', '--libdir'
|
173
|
-
libdir = ARGV.req_arg
|
174
|
-
when '-b', '--bindir'
|
175
|
-
bindir = ARGV.req_arg
|
176
|
-
when '-r', '--ruby'
|
177
|
-
$ruby = ARGV.req_arg
|
178
|
-
when '-n', '--no_linkify'
|
179
|
-
no_linkify = true
|
180
|
-
when '-s', '--sudo'
|
181
|
-
sudo = 'sudo'
|
182
|
-
when '-h', '--help'
|
183
|
-
help = true
|
184
|
-
else
|
185
|
-
raise "unknown switch #{switch.dump}"
|
186
|
-
end
|
187
|
-
end
|
188
|
-
rescue
|
189
|
-
STDERR.puts $!.to_s
|
190
|
-
STDERR.puts usage
|
191
|
-
exit 1
|
192
|
-
end
|
193
|
-
|
194
|
-
if help
|
195
|
-
STDOUT.puts usage
|
196
|
-
exit
|
197
|
-
end
|
198
|
-
|
199
|
-
system "#{ sudo } #{ $ruby } pre-install.rb" if test(?s, 'pre-install.rb')
|
200
|
-
|
201
|
-
unless no_linkify
|
202
|
-
linked = linkify('lib') + linkify('bin')
|
203
|
-
end
|
204
|
-
|
205
|
-
system "#{ $ruby } extconf.rb && make && #{ sudo } make install" if test(?s, 'extconf.rb')
|
206
|
-
|
207
|
-
install_rb(LIBDIR, libdir, LIBDIR_MODE)
|
208
|
-
install_rb(BINDIR, bindir, BINDIR_MODE, bin=true)
|
209
|
-
|
210
|
-
if linked
|
211
|
-
linked.each{|path| File::rm_f path}
|
212
|
-
end
|
213
|
-
|
214
|
-
system "#{ sudo } #{ $ruby } post-install.rb" if test(?s, 'post-install.rb')
|