forkoff 0.0.0 → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY +34 -0
- data/README +62 -25
- data/lib/forkoff.rb +47 -27
- data/samples/b.rb +2 -4
- data/samples/c.rb +3 -3
- metadata +3 -2
data/HISTORY
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
|
2
|
+
0.0.1
|
3
|
+
|
4
|
+
- updated to use producer threds pushing onto a SizedQueue for each consumer
|
5
|
+
channel. in this way the producers do not build up a massize parllel data
|
6
|
+
structure but provide data to the consumers only as fast as they can fork
|
7
|
+
and proccess it. basically for a 4 process run you'll end up with 4
|
8
|
+
channels of size 1 between 4 produces and 4 consumers, each consumer is a
|
9
|
+
thread popping of jobs, forking, and yielding results.
|
10
|
+
|
11
|
+
- removed use of Queue for capturing the output. now it's simply an array
|
12
|
+
of arrays which removed some sync overhead.
|
13
|
+
|
14
|
+
- you can configure the number of processes globally with
|
15
|
+
|
16
|
+
Forkoff.default['proccess'] = 4
|
17
|
+
|
18
|
+
- you can now pass either an options hash
|
19
|
+
|
20
|
+
forkoff( :processes => 2 ) ...
|
21
|
+
|
22
|
+
or plain vanilla number
|
23
|
+
|
24
|
+
forkoff( 2 ) ...
|
25
|
+
|
26
|
+
to the forkoff call
|
27
|
+
|
28
|
+
- default number of processes is 8, not 2
|
29
|
+
|
30
|
+
|
31
|
+
0.0.0
|
32
|
+
|
33
|
+
initial version
|
34
|
+
|
data/README
CHANGED
@@ -18,7 +18,7 @@ DESCRIPTION
|
|
18
18
|
|
19
19
|
forkoff works for any enumerable object, iterating a code block to run in a
|
20
20
|
child process and collecting the results. forkoff can limit the number of
|
21
|
-
child processes which is, by default,
|
21
|
+
child processes which is, by default, 2.
|
22
22
|
|
23
23
|
SAMPLES
|
24
24
|
|
@@ -37,8 +37,8 @@ SAMPLES
|
|
37
37
|
|
38
38
|
~ > ruby samples/a.rb
|
39
39
|
|
40
|
-
hey from
|
41
|
-
you from
|
40
|
+
hey from 19511
|
41
|
+
you from 19512
|
42
42
|
|
43
43
|
|
44
44
|
<========< samples/b.rb >========>
|
@@ -46,7 +46,8 @@ SAMPLES
|
|
46
46
|
~ > cat samples/b.rb
|
47
47
|
|
48
48
|
#
|
49
|
-
# for example, this takes only
|
49
|
+
# for example, this takes only 4 seconds or so to complete (8 iterations
|
50
|
+
# running in two processes = twice as fast)
|
50
51
|
#
|
51
52
|
|
52
53
|
require 'forkoff'
|
@@ -55,11 +56,8 @@ SAMPLES
|
|
55
56
|
|
56
57
|
results =
|
57
58
|
(0..7).forkoff do |i|
|
58
|
-
|
59
59
|
sleep 1
|
60
|
-
|
61
60
|
i ** 2
|
62
|
-
|
63
61
|
end
|
64
62
|
|
65
63
|
b = Time.now.to_f
|
@@ -71,7 +69,7 @@ SAMPLES
|
|
71
69
|
|
72
70
|
~ > ruby samples/b.rb
|
73
71
|
|
74
|
-
elapsed:
|
72
|
+
elapsed: 4.19796895980835
|
75
73
|
results: [0, 1, 4, 9, 16, 25, 36, 49]
|
76
74
|
|
77
75
|
|
@@ -82,9 +80,9 @@ SAMPLES
|
|
82
80
|
#
|
83
81
|
# forkoff does *NOT* spawn processes in batches, waiting for each batch to
|
84
82
|
# complete. rather, it keeps a certain number of processes busy until all
|
85
|
-
# results have been gathered. in otherwords the following will ensure that
|
83
|
+
# results have been gathered. in otherwords the following will ensure that 3
|
86
84
|
# processes are running at all times, until the list is complete. note that
|
87
|
-
# the following will take about
|
85
|
+
# the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
|
88
86
|
#
|
89
87
|
|
90
88
|
require 'forkoff'
|
@@ -94,7 +92,7 @@ SAMPLES
|
|
94
92
|
a = Time.now.to_f
|
95
93
|
|
96
94
|
pstrees =
|
97
|
-
%w( a b c d ).forkoff! :processes =>
|
95
|
+
%w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
|
98
96
|
sleep 1
|
99
97
|
{ letter => ` pstree -l 2 #{ pid } ` }
|
100
98
|
end
|
@@ -116,31 +114,70 @@ SAMPLES
|
|
116
114
|
~ > ruby samples/c.rb
|
117
115
|
|
118
116
|
|
119
|
-
pid:
|
120
|
-
elapsed:
|
117
|
+
pid: 19526
|
118
|
+
elapsed: 3.39831399917603
|
121
119
|
|
122
120
|
---
|
123
121
|
a: |
|
124
|
-
-+-
|
125
|
-
|-+-
|
126
|
-
|
122
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
123
|
+
|-+- 19527 ahoward ruby -Ilib samples/c.rb
|
124
|
+
|-+- 19528 ahoward ruby -Ilib samples/c.rb
|
125
|
+
\-+- 19529 ahoward ruby -Ilib samples/c.rb
|
127
126
|
|
128
127
|
---
|
129
128
|
b: |
|
130
|
-
-+-
|
131
|
-
|-+-
|
132
|
-
|
129
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
130
|
+
|-+- 19527 ahoward ruby -Ilib samples/c.rb
|
131
|
+
|-+- 19528 ahoward ruby -Ilib samples/c.rb
|
132
|
+
\-+- 19529 ahoward ruby -Ilib samples/c.rb
|
133
133
|
|
134
134
|
---
|
135
135
|
c: |
|
136
|
-
-+-
|
137
|
-
|-+-
|
138
|
-
|
136
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
137
|
+
|-+- 19527 ahoward (ruby)
|
138
|
+
|-+- 19528 ahoward (ruby)
|
139
|
+
\-+- 19529 ahoward ruby -Ilib samples/c.rb
|
139
140
|
|
140
141
|
---
|
141
142
|
d: |
|
142
|
-
-+-
|
143
|
-
|-+-
|
144
|
-
|
143
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
144
|
+
|-+- 19536 ahoward ruby -Ilib samples/c.rb
|
145
|
+
|-+- 19537 ahoward (ruby)
|
146
|
+
\--- 19538 ahoward ruby -Ilib samples/c.rb
|
147
|
+
|
148
|
+
---
|
149
|
+
e: |
|
150
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
151
|
+
|-+- 19536 ahoward ruby -Ilib samples/c.rb
|
152
|
+
|-+- 19537 ahoward ruby -Ilib samples/c.rb
|
153
|
+
\--- 19538 ahoward ruby -Ilib samples/c.rb
|
154
|
+
|
155
|
+
---
|
156
|
+
f: |
|
157
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
158
|
+
|--- 19536 ahoward (ruby)
|
159
|
+
|-+- 19538 ahoward ruby -Ilib samples/c.rb
|
160
|
+
\--- 19543 ahoward ruby -Ilib samples/c.rb
|
161
|
+
|
162
|
+
---
|
163
|
+
g: |
|
164
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
165
|
+
|--- 19543 ahoward (ruby)
|
166
|
+
|-+- 19546 ahoward ruby -Ilib samples/c.rb
|
167
|
+
\--- 19547 ahoward ruby -Ilib samples/c.rb
|
168
|
+
|
169
|
+
---
|
170
|
+
h: |
|
171
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
172
|
+
|-+- 19543 ahoward ruby -Ilib samples/c.rb
|
173
|
+
|--- 19546 ahoward ruby -Ilib samples/c.rb
|
174
|
+
\--- 19547 ahoward ruby -Ilib samples/c.rb
|
175
|
+
|
176
|
+
---
|
177
|
+
i: |
|
178
|
+
-+- 19526 ahoward ruby -Ilib samples/c.rb
|
179
|
+
|-+- 19543 ahoward ruby -Ilib samples/c.rb
|
180
|
+
|-+- 19546 ahoward ruby -Ilib samples/c.rb
|
181
|
+
\-+- 19547 ahoward ruby -Ilib samples/c.rb
|
145
182
|
|
146
183
|
|
data/lib/forkoff.rb
CHANGED
@@ -1,26 +1,42 @@
|
|
1
1
|
require 'thread'
|
2
2
|
|
3
|
+
module Forkoff
|
4
|
+
VERSION = '0.0.1'
|
5
|
+
def self::version() Forkoff::VERSION end
|
6
|
+
|
7
|
+
Done = Object.new
|
8
|
+
def self::done() Forkoff::Done end
|
9
|
+
|
10
|
+
@default = {
|
11
|
+
'processes' => 2
|
12
|
+
}
|
13
|
+
|
14
|
+
class << self
|
15
|
+
attr_accessor 'default'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
3
19
|
module Enumerable
|
4
20
|
|
5
21
|
def forkoff options = {}, &block
|
6
|
-
|
7
|
-
|
8
|
-
qs = Array.new(n){
|
9
|
-
results = Array.new(n){
|
22
|
+
options = { 'processes' => Integer(options) } unless Hash === options
|
23
|
+
n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
|
24
|
+
qs = Array.new(n){ SizedQueue.new 1 }
|
25
|
+
results = Array.new(n){ [] }
|
10
26
|
|
11
27
|
#
|
12
28
|
# consumers
|
13
29
|
#
|
14
|
-
|
30
|
+
consumers = []
|
15
31
|
|
16
32
|
n.times do |i|
|
17
33
|
thread =
|
18
|
-
Thread.new
|
34
|
+
Thread.new do
|
19
35
|
Thread.current.abort_on_exception = true
|
20
36
|
|
21
37
|
loop do
|
22
38
|
value = qs[i].pop
|
23
|
-
break if value == done
|
39
|
+
break if value == Forkoff.done
|
24
40
|
args, index = value
|
25
41
|
|
26
42
|
r, w = IO.pipe
|
@@ -44,48 +60,52 @@ module Enumerable
|
|
44
60
|
Process.waitpid pid
|
45
61
|
end
|
46
62
|
|
47
|
-
results[i].push( done )
|
63
|
+
results[i].push( Forkoff.done )
|
48
64
|
end
|
49
65
|
|
50
|
-
|
66
|
+
consumers << thread
|
51
67
|
end
|
52
68
|
|
53
69
|
#
|
54
|
-
#
|
70
|
+
# producers
|
55
71
|
#
|
56
|
-
|
57
|
-
qs[ i.modulo(qs.size) ].push( [args, i] )
|
58
|
-
end
|
72
|
+
producers = []
|
59
73
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
74
|
+
n.times do |i|
|
75
|
+
thread = Thread.new do
|
76
|
+
Thread.current.abort_on_exception = true
|
77
|
+
each_with_index do |args, j|
|
78
|
+
every_nth = j.modulo(n) == i
|
79
|
+
next unless every_nth
|
80
|
+
qs[ j.modulo(n) ].push( [args, j] )
|
81
|
+
end
|
82
|
+
qs[ i ].push( Forkoff.done )
|
83
|
+
end
|
84
|
+
|
85
|
+
producers << thread
|
65
86
|
end
|
66
87
|
|
67
88
|
#
|
68
|
-
# wait for all
|
89
|
+
# wait for all consumers to complete
|
69
90
|
#
|
70
|
-
|
91
|
+
consumers.each do |t|
|
71
92
|
t.value
|
72
93
|
end
|
73
94
|
|
74
95
|
#
|
75
96
|
# gather results
|
76
97
|
#
|
77
|
-
|
98
|
+
returned = []
|
78
99
|
|
79
|
-
results.each do |
|
80
|
-
|
81
|
-
value
|
82
|
-
break if value == done
|
100
|
+
results.each do |set|
|
101
|
+
set.each do |value|
|
102
|
+
break if value == Forkoff.done
|
83
103
|
result, index = value
|
84
|
-
|
104
|
+
returned[index] = result
|
85
105
|
end
|
86
106
|
end
|
87
107
|
|
88
|
-
|
108
|
+
returned
|
89
109
|
end
|
90
110
|
|
91
111
|
alias_method 'forkoff!', 'forkoff'
|
data/samples/b.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
#
|
2
|
-
# for example, this takes only
|
2
|
+
# for example, this takes only 4 seconds or so to complete (8 iterations
|
3
|
+
# running in two processes = twice as fast)
|
3
4
|
#
|
4
5
|
|
5
6
|
require 'forkoff'
|
@@ -8,11 +9,8 @@
|
|
8
9
|
|
9
10
|
results =
|
10
11
|
(0..7).forkoff do |i|
|
11
|
-
|
12
12
|
sleep 1
|
13
|
-
|
14
13
|
i ** 2
|
15
|
-
|
16
14
|
end
|
17
15
|
|
18
16
|
b = Time.now.to_f
|
data/samples/c.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
#
|
2
2
|
# forkoff does *NOT* spawn processes in batches, waiting for each batch to
|
3
3
|
# complete. rather, it keeps a certain number of processes busy until all
|
4
|
-
# results have been gathered. in otherwords the following will ensure that
|
4
|
+
# results have been gathered. in otherwords the following will ensure that 3
|
5
5
|
# processes are running at all times, until the list is complete. note that
|
6
|
-
# the following will take about
|
6
|
+
# the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
|
7
7
|
#
|
8
8
|
|
9
9
|
require 'forkoff'
|
@@ -13,7 +13,7 @@ pid = Process.pid
|
|
13
13
|
a = Time.now.to_f
|
14
14
|
|
15
15
|
pstrees =
|
16
|
-
%w( a b c d ).forkoff! :processes =>
|
16
|
+
%w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
|
17
17
|
sleep 1
|
18
18
|
{ letter => ` pstree -l 2 #{ pid } ` }
|
19
19
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: forkoff
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ara T. Howard
|
@@ -9,7 +9,7 @@ autorequire: forkoff
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2008-04-
|
12
|
+
date: 2008-04-21 00:00:00 -06:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -24,6 +24,7 @@ extra_rdoc_files: []
|
|
24
24
|
files:
|
25
25
|
- gemspec.rb
|
26
26
|
- gen_readme.rb
|
27
|
+
- HISTORY
|
27
28
|
- install.rb
|
28
29
|
- lib
|
29
30
|
- lib/forkoff.rb
|