forkoff 0.0.0 → 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. data/HISTORY +34 -0
  2. data/README +62 -25
  3. data/lib/forkoff.rb +47 -27
  4. data/samples/b.rb +2 -4
  5. data/samples/c.rb +3 -3
  6. metadata +3 -2
data/HISTORY ADDED
@@ -0,0 +1,34 @@
1
+
2
+ 0.0.1
3
+
4
+ - updated to use producer threds pushing onto a SizedQueue for each consumer
5
+ channel. in this way the producers do not build up a massize parllel data
6
+ structure but provide data to the consumers only as fast as they can fork
7
+ and proccess it. basically for a 4 process run you'll end up with 4
8
+ channels of size 1 between 4 produces and 4 consumers, each consumer is a
9
+ thread popping of jobs, forking, and yielding results.
10
+
11
+ - removed use of Queue for capturing the output. now it's simply an array
12
+ of arrays which removed some sync overhead.
13
+
14
+ - you can configure the number of processes globally with
15
+
16
+ Forkoff.default['proccess'] = 4
17
+
18
+ - you can now pass either an options hash
19
+
20
+ forkoff( :processes => 2 ) ...
21
+
22
+ or plain vanilla number
23
+
24
+ forkoff( 2 ) ...
25
+
26
+ to the forkoff call
27
+
28
+ - default number of processes is 8, not 2
29
+
30
+
31
+ 0.0.0
32
+
33
+ initial version
34
+
data/README CHANGED
@@ -18,7 +18,7 @@ DESCRIPTION
18
18
 
19
19
  forkoff works for any enumerable object, iterating a code block to run in a
20
20
  child process and collecting the results. forkoff can limit the number of
21
- child processes which is, by default, 8.
21
+ child processes which is, by default, 2.
22
22
 
23
23
  SAMPLES
24
24
 
@@ -37,8 +37,8 @@ SAMPLES
37
37
 
38
38
  ~ > ruby samples/a.rb
39
39
 
40
- hey from 3239
41
- you from 3240
40
+ hey from 19511
41
+ you from 19512
42
42
 
43
43
 
44
44
  <========< samples/b.rb >========>
@@ -46,7 +46,8 @@ SAMPLES
46
46
  ~ > cat samples/b.rb
47
47
 
48
48
  #
49
- # for example, this takes only 1 second or so to complete
49
+ # for example, this takes only 4 seconds or so to complete (8 iterations
50
+ # running in two processes = twice as fast)
50
51
  #
51
52
 
52
53
  require 'forkoff'
@@ -55,11 +56,8 @@ SAMPLES
55
56
 
56
57
  results =
57
58
  (0..7).forkoff do |i|
58
-
59
59
  sleep 1
60
-
61
60
  i ** 2
62
-
63
61
  end
64
62
 
65
63
  b = Time.now.to_f
@@ -71,7 +69,7 @@ SAMPLES
71
69
 
72
70
  ~ > ruby samples/b.rb
73
71
 
74
- elapsed: 1.07044386863708
72
+ elapsed: 4.19796895980835
75
73
  results: [0, 1, 4, 9, 16, 25, 36, 49]
76
74
 
77
75
 
@@ -82,9 +80,9 @@ SAMPLES
82
80
  #
83
81
  # forkoff does *NOT* spawn processes in batches, waiting for each batch to
84
82
  # complete. rather, it keeps a certain number of processes busy until all
85
- # results have been gathered. in otherwords the following will ensure that 2
83
+ # results have been gathered. in otherwords the following will ensure that 3
86
84
  # processes are running at all times, until the list is complete. note that
87
- # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
85
+ # the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
88
86
  #
89
87
 
90
88
  require 'forkoff'
@@ -94,7 +92,7 @@ SAMPLES
94
92
  a = Time.now.to_f
95
93
 
96
94
  pstrees =
97
- %w( a b c d ).forkoff! :processes => 2 do |letter|
95
+ %w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
98
96
  sleep 1
99
97
  { letter => ` pstree -l 2 #{ pid } ` }
100
98
  end
@@ -116,31 +114,70 @@ SAMPLES
116
114
  ~ > ruby samples/c.rb
117
115
 
118
116
 
119
- pid: 3254
120
- elapsed: 2.12998485565186
117
+ pid: 19526
118
+ elapsed: 3.39831399917603
121
119
 
122
120
  ---
123
121
  a: |
124
- -+- 03254 ahoward ruby -Ilib samples/c.rb
125
- |-+- 03255 ahoward ruby -Ilib samples/c.rb
126
- \-+- 03256 ahoward ruby -Ilib samples/c.rb
122
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
123
+ |-+- 19527 ahoward ruby -Ilib samples/c.rb
124
+ |-+- 19528 ahoward ruby -Ilib samples/c.rb
125
+ \-+- 19529 ahoward ruby -Ilib samples/c.rb
127
126
 
128
127
  ---
129
128
  b: |
130
- -+- 03254 ahoward ruby -Ilib samples/c.rb
131
- |-+- 03255 ahoward ruby -Ilib samples/c.rb
132
- \-+- 03256 ahoward ruby -Ilib samples/c.rb
129
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
130
+ |-+- 19527 ahoward ruby -Ilib samples/c.rb
131
+ |-+- 19528 ahoward ruby -Ilib samples/c.rb
132
+ \-+- 19529 ahoward ruby -Ilib samples/c.rb
133
133
 
134
134
  ---
135
135
  c: |
136
- -+- 03254 ahoward ruby -Ilib samples/c.rb
137
- |-+- 03261 ahoward (ruby)
138
- \-+- 03262 ahoward ruby -Ilib samples/c.rb
136
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
137
+ |-+- 19527 ahoward (ruby)
138
+ |-+- 19528 ahoward (ruby)
139
+ \-+- 19529 ahoward ruby -Ilib samples/c.rb
139
140
 
140
141
  ---
141
142
  d: |
142
- -+- 03254 ahoward ruby -Ilib samples/c.rb
143
- |-+- 03261 ahoward ruby -Ilib samples/c.rb
144
- \-+- 03262 ahoward ruby -Ilib samples/c.rb
143
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
144
+ |-+- 19536 ahoward ruby -Ilib samples/c.rb
145
+ |-+- 19537 ahoward (ruby)
146
+ \--- 19538 ahoward ruby -Ilib samples/c.rb
147
+
148
+ ---
149
+ e: |
150
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
151
+ |-+- 19536 ahoward ruby -Ilib samples/c.rb
152
+ |-+- 19537 ahoward ruby -Ilib samples/c.rb
153
+ \--- 19538 ahoward ruby -Ilib samples/c.rb
154
+
155
+ ---
156
+ f: |
157
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
158
+ |--- 19536 ahoward (ruby)
159
+ |-+- 19538 ahoward ruby -Ilib samples/c.rb
160
+ \--- 19543 ahoward ruby -Ilib samples/c.rb
161
+
162
+ ---
163
+ g: |
164
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
165
+ |--- 19543 ahoward (ruby)
166
+ |-+- 19546 ahoward ruby -Ilib samples/c.rb
167
+ \--- 19547 ahoward ruby -Ilib samples/c.rb
168
+
169
+ ---
170
+ h: |
171
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
172
+ |-+- 19543 ahoward ruby -Ilib samples/c.rb
173
+ |--- 19546 ahoward ruby -Ilib samples/c.rb
174
+ \--- 19547 ahoward ruby -Ilib samples/c.rb
175
+
176
+ ---
177
+ i: |
178
+ -+- 19526 ahoward ruby -Ilib samples/c.rb
179
+ |-+- 19543 ahoward ruby -Ilib samples/c.rb
180
+ |-+- 19546 ahoward ruby -Ilib samples/c.rb
181
+ \-+- 19547 ahoward ruby -Ilib samples/c.rb
145
182
 
146
183
 
@@ -1,26 +1,42 @@
1
1
  require 'thread'
2
2
 
3
+ module Forkoff
4
+ VERSION = '0.0.1'
5
+ def self::version() Forkoff::VERSION end
6
+
7
+ Done = Object.new
8
+ def self::done() Forkoff::Done end
9
+
10
+ @default = {
11
+ 'processes' => 2
12
+ }
13
+
14
+ class << self
15
+ attr_accessor 'default'
16
+ end
17
+ end
18
+
3
19
  module Enumerable
4
20
 
5
21
  def forkoff options = {}, &block
6
- n = Integer( options['processes'] || options[:processes] || 8 )
7
- done = Object.new
8
- qs = Array.new(n){ Queue.new }
9
- results = Array.new(n){ Queue.new }
22
+ options = { 'processes' => Integer(options) } unless Hash === options
23
+ n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
24
+ qs = Array.new(n){ SizedQueue.new 1 }
25
+ results = Array.new(n){ [] }
10
26
 
11
27
  #
12
28
  # consumers
13
29
  #
14
- threads = []
30
+ consumers = []
15
31
 
16
32
  n.times do |i|
17
33
  thread =
18
- Thread.new(i) do |i|
34
+ Thread.new do
19
35
  Thread.current.abort_on_exception = true
20
36
 
21
37
  loop do
22
38
  value = qs[i].pop
23
- break if value == done
39
+ break if value == Forkoff.done
24
40
  args, index = value
25
41
 
26
42
  r, w = IO.pipe
@@ -44,48 +60,52 @@ module Enumerable
44
60
  Process.waitpid pid
45
61
  end
46
62
 
47
- results[i].push( done )
63
+ results[i].push( Forkoff.done )
48
64
  end
49
65
 
50
- threads << thread
66
+ consumers << thread
51
67
  end
52
68
 
53
69
  #
54
- # producer
70
+ # producers
55
71
  #
56
- each_with_index do |args, i|
57
- qs[ i.modulo(qs.size) ].push( [args, i] )
58
- end
72
+ producers = []
59
73
 
60
- #
61
- # mark the end of each queue
62
- #
63
- qs.each do |q|
64
- q.push done
74
+ n.times do |i|
75
+ thread = Thread.new do
76
+ Thread.current.abort_on_exception = true
77
+ each_with_index do |args, j|
78
+ every_nth = j.modulo(n) == i
79
+ next unless every_nth
80
+ qs[ j.modulo(n) ].push( [args, j] )
81
+ end
82
+ qs[ i ].push( Forkoff.done )
83
+ end
84
+
85
+ producers << thread
65
86
  end
66
87
 
67
88
  #
68
- # wait for all threads to complete
89
+ # wait for all consumers to complete
69
90
  #
70
- threads.each do |t|
91
+ consumers.each do |t|
71
92
  t.value
72
93
  end
73
94
 
74
95
  #
75
96
  # gather results
76
97
  #
77
- list = []
98
+ returned = []
78
99
 
79
- results.each do |q|
80
- loop do
81
- value = q.pop
82
- break if value == done
100
+ results.each do |set|
101
+ set.each do |value|
102
+ break if value == Forkoff.done
83
103
  result, index = value
84
- list[index] = result
104
+ returned[index] = result
85
105
  end
86
106
  end
87
107
 
88
- list
108
+ returned
89
109
  end
90
110
 
91
111
  alias_method 'forkoff!', 'forkoff'
@@ -1,5 +1,6 @@
1
1
  #
2
- # for example, this takes only 1 second or so to complete
2
+ # for example, this takes only 4 seconds or so to complete (8 iterations
3
+ # running in two processes = twice as fast)
3
4
  #
4
5
 
5
6
  require 'forkoff'
@@ -8,11 +9,8 @@
8
9
 
9
10
  results =
10
11
  (0..7).forkoff do |i|
11
-
12
12
  sleep 1
13
-
14
13
  i ** 2
15
-
16
14
  end
17
15
 
18
16
  b = Time.now.to_f
@@ -1,9 +1,9 @@
1
1
  #
2
2
  # forkoff does *NOT* spawn processes in batches, waiting for each batch to
3
3
  # complete. rather, it keeps a certain number of processes busy until all
4
- # results have been gathered. in otherwords the following will ensure that 2
4
+ # results have been gathered. in otherwords the following will ensure that 3
5
5
  # processes are running at all times, until the list is complete. note that
6
- # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
6
+ # the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
7
7
  #
8
8
 
9
9
  require 'forkoff'
@@ -13,7 +13,7 @@ pid = Process.pid
13
13
  a = Time.now.to_f
14
14
 
15
15
  pstrees =
16
- %w( a b c d ).forkoff! :processes => 2 do |letter|
16
+ %w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
17
17
  sleep 1
18
18
  { letter => ` pstree -l 2 #{ pid } ` }
19
19
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: forkoff
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ara T. Howard
@@ -9,7 +9,7 @@ autorequire: forkoff
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2008-04-17 00:00:00 -06:00
12
+ date: 2008-04-21 00:00:00 -06:00
13
13
  default_executable:
14
14
  dependencies: []
15
15
 
@@ -24,6 +24,7 @@ extra_rdoc_files: []
24
24
  files:
25
25
  - gemspec.rb
26
26
  - gen_readme.rb
27
+ - HISTORY
27
28
  - install.rb
28
29
  - lib
29
30
  - lib/forkoff.rb