parallel 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +8 -10
- data/VERSION +1 -1
- data/lib/parallel.rb +35 -48
- data/parallel.gemspec +2 -2
- data/spec/cases/map_with_processes_and_exceptions.rb +4 -7
- data/spec/cases/map_with_threads_and_exceptions.rb +3 -7
- data/spec/parallel_spec.rb +25 -20
- metadata +4 -4
data/Gemfile.lock
CHANGED
@@ -10,16 +10,14 @@ GEM
|
|
10
10
|
rubyforge (>= 2.0.0)
|
11
11
|
json_pure (1.4.6)
|
12
12
|
rake (0.8.7)
|
13
|
-
rspec (2.0
|
14
|
-
rspec-core (~> 2.0
|
15
|
-
rspec-expectations (~> 2.0
|
16
|
-
rspec-mocks (~> 2.0
|
17
|
-
rspec-core (2.
|
18
|
-
rspec-expectations (2.0
|
19
|
-
diff-lcs (
|
20
|
-
rspec-mocks (2.0
|
21
|
-
rspec-core (~> 2.0.1)
|
22
|
-
rspec-expectations (~> 2.0.1)
|
13
|
+
rspec (2.6.0)
|
14
|
+
rspec-core (~> 2.6.0)
|
15
|
+
rspec-expectations (~> 2.6.0)
|
16
|
+
rspec-mocks (~> 2.6.0)
|
17
|
+
rspec-core (2.6.4)
|
18
|
+
rspec-expectations (2.6.0)
|
19
|
+
diff-lcs (~> 1.1.2)
|
20
|
+
rspec-mocks (2.6.0)
|
23
21
|
rubyforge (2.0.4)
|
24
22
|
json_pure (>= 1.1.7)
|
25
23
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.8
|
data/lib/parallel.rb
CHANGED
@@ -122,56 +122,45 @@ class Parallel
|
|
122
122
|
end
|
123
123
|
|
124
124
|
def self.work_in_processes(items, options, &blk)
|
125
|
-
workers = Array.new(options[:count]).map{ worker(items, options, &blk) }
|
126
|
-
Parallel.kill_on_ctrl_c(workers.map{|worker| worker[:pid] })
|
127
|
-
|
128
125
|
current_index = -1
|
126
|
+
results = []
|
127
|
+
pids = []
|
128
|
+
exception = nil
|
129
129
|
|
130
|
-
|
131
|
-
workers.each do |worker|
|
132
|
-
write_to_pipe(worker[:write], current_index += 1)
|
133
|
-
end
|
130
|
+
Parallel.kill_on_ctrl_c(pids)
|
134
131
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
132
|
+
in_threads(options[:count]) do |i|
|
133
|
+
x = i
|
134
|
+
worker = worker(items, options, &blk)
|
135
|
+
pids[i] = worker[:pid]
|
139
136
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
result[result_index] = output
|
154
|
-
|
155
|
-
# give worker next item
|
156
|
-
next_index = Thread.exclusive{ current_index += 1 }
|
157
|
-
break if next_index >= items.size
|
158
|
-
write_to_pipe(worker[:write], next_index)
|
137
|
+
begin
|
138
|
+
loop do
|
139
|
+
break if exception
|
140
|
+
index = Thread.exclusive{ current_index += 1 }
|
141
|
+
break if index >= items.size
|
142
|
+
|
143
|
+
write_to_pipe(worker[:write], index)
|
144
|
+
output = decode(worker[:read].gets.chomp)
|
145
|
+
|
146
|
+
if ExceptionWrapper === output
|
147
|
+
exception = output.exception
|
148
|
+
else
|
149
|
+
results[index] = output
|
159
150
|
end
|
160
|
-
ensure
|
161
|
-
worker[:read].close
|
162
|
-
worker[:write].close
|
163
151
|
end
|
152
|
+
ensure
|
153
|
+
worker[:read].close
|
154
|
+
worker[:write].close
|
155
|
+
|
156
|
+
# if it goes zombie, rather wait here to be able to debug
|
157
|
+
wait_for_process worker[:pid]
|
164
158
|
end
|
165
159
|
end
|
166
160
|
|
167
|
-
wait_for_threads(listener_threads)
|
168
|
-
|
169
|
-
# if they go zombie, rather wait here to be able to debug
|
170
|
-
wait_for_processes(workers.map{|worker| worker[:pid] })
|
171
|
-
|
172
161
|
raise exception if exception
|
173
162
|
|
174
|
-
|
163
|
+
results
|
175
164
|
end
|
176
165
|
|
177
166
|
def self.worker(items, options, &block)
|
@@ -208,7 +197,7 @@ class Parallel
|
|
208
197
|
rescue Exception => e
|
209
198
|
result = ExceptionWrapper.new(e)
|
210
199
|
end
|
211
|
-
write_to_pipe(write,
|
200
|
+
write_to_pipe(write, result)
|
212
201
|
end
|
213
202
|
end
|
214
203
|
|
@@ -226,13 +215,11 @@ class Parallel
|
|
226
215
|
end
|
227
216
|
end
|
228
217
|
|
229
|
-
def self.
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
# process died
|
235
|
-
end
|
218
|
+
def self.wait_for_process(pid)
|
219
|
+
begin
|
220
|
+
Process.wait(pid)
|
221
|
+
rescue Interrupt
|
222
|
+
# process died
|
236
223
|
end
|
237
224
|
end
|
238
225
|
|
@@ -259,7 +246,7 @@ class Parallel
|
|
259
246
|
def self.kill_on_ctrl_c(pids)
|
260
247
|
Signal.trap :SIGINT do
|
261
248
|
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
262
|
-
pids.each { |pid| Process.kill(:KILL, pid) }
|
249
|
+
pids.each { |pid| Process.kill(:KILL, pid) if pid }
|
263
250
|
exit 1 # Quit with 'failed' signal
|
264
251
|
end
|
265
252
|
end
|
data/parallel.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{parallel}
|
8
|
-
s.version = "0.5.
|
8
|
+
s.version = "0.5.8"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Michael Grosser"]
|
12
|
-
s.date = %q{2011-08-
|
12
|
+
s.date = %q{2011-08-18}
|
13
13
|
s.email = %q{grosser.michael@gmail.com}
|
14
14
|
s.files = [
|
15
15
|
"Gemfile",
|
@@ -1,16 +1,13 @@
|
|
1
1
|
require File.expand_path('spec/spec_helper')
|
2
2
|
|
3
|
-
class Parallel
|
4
|
-
def self.wait_for_threads(threads)
|
5
|
-
print ' all joined'
|
6
|
-
end
|
7
|
-
end
|
8
|
-
|
9
3
|
begin
|
10
4
|
Parallel.map(1..100, :in_processes => 4) do |x|
|
5
|
+
sleep 0.1 # so all processes get started
|
11
6
|
print x
|
12
7
|
raise 'foo' if x == 1
|
8
|
+
sleep 0.1 # so no now work gets queued before exception is raised
|
9
|
+
x
|
13
10
|
end
|
14
11
|
rescue
|
15
12
|
print ' raised'
|
16
|
-
end
|
13
|
+
end
|
@@ -1,16 +1,12 @@
|
|
1
1
|
require File.expand_path('spec/spec_helper')
|
2
2
|
|
3
|
-
class Parallel
|
4
|
-
def self.wait_for_threads(threads)
|
5
|
-
print ' all joined'
|
6
|
-
end
|
7
|
-
end
|
8
|
-
|
9
3
|
begin
|
10
4
|
Parallel.map(1..100, :in_threads => 4) do |x|
|
5
|
+
sleep 0.1 # so all processes get started
|
11
6
|
print x
|
12
7
|
raise 'foo' if x == 1
|
8
|
+
sleep 0.1 # so no now work gets queued before exception is raised
|
13
9
|
end
|
14
10
|
rescue
|
15
11
|
print ' raised'
|
16
|
-
end
|
12
|
+
end
|
data/spec/parallel_spec.rb
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
require File.expand_path('spec/spec_helper')
|
2
2
|
|
3
3
|
describe Parallel do
|
4
|
+
def time_taken
|
5
|
+
t = Time.now.to_f
|
6
|
+
yield
|
7
|
+
Time.now.to_f - t
|
8
|
+
end
|
4
9
|
|
5
10
|
describe :processor_count do
|
6
11
|
it "returns a number" do
|
@@ -37,25 +42,25 @@ describe Parallel do
|
|
37
42
|
end
|
38
43
|
|
39
44
|
it "kills the processes when the main process gets killed through ctrl+c" do
|
40
|
-
|
45
|
+
time_taken{
|
41
46
|
lambda{
|
42
47
|
Thread.new do
|
43
48
|
`ruby spec/cases/parallel_start_and_kill.rb`
|
44
49
|
end
|
45
50
|
sleep 1
|
46
51
|
running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) }
|
47
|
-
pid_index = running_processes.detect{ |
|
48
|
-
parent_pid = running_processes.detect{ |
|
52
|
+
pid_index = running_processes.detect{ |p| p.include?("UID") }.index("UID") + 1
|
53
|
+
parent_pid = running_processes.detect{ |p| p.include?("spec/cases/parallel_start_and_kill.rb") }[pid_index]
|
49
54
|
`kill -2 #{parent_pid}` #simulates Ctrl+c
|
50
55
|
sleep 1
|
51
56
|
}.should_not change{`ps`.split("\n").size}
|
52
|
-
|
57
|
+
}.should <= 3
|
53
58
|
end
|
54
59
|
|
55
60
|
it "saves time" do
|
56
|
-
|
57
|
-
|
58
|
-
|
61
|
+
time_taken{
|
62
|
+
`ruby spec/cases/parallel_sleeping_2.rb`
|
63
|
+
}.should < 3.5
|
59
64
|
end
|
60
65
|
|
61
66
|
it "raises when one of the processes raises" do
|
@@ -73,9 +78,9 @@ describe Parallel do
|
|
73
78
|
|
74
79
|
describe :in_threads do
|
75
80
|
it "saves time" do
|
76
|
-
|
77
|
-
|
78
|
-
|
81
|
+
time_taken{
|
82
|
+
Parallel.in_threads(3){ sleep 2 }
|
83
|
+
}.should < 3
|
79
84
|
end
|
80
85
|
|
81
86
|
it "does not create new processes" do
|
@@ -93,9 +98,9 @@ describe Parallel do
|
|
93
98
|
|
94
99
|
describe :map do
|
95
100
|
it "saves time" do
|
96
|
-
|
101
|
+
time_taken{
|
97
102
|
`ruby spec/cases/parallel_map_sleeping.rb`
|
98
|
-
|
103
|
+
}.should <= 3.5
|
99
104
|
end
|
100
105
|
|
101
106
|
it "executes with given parameters" do
|
@@ -103,9 +108,9 @@ describe Parallel do
|
|
103
108
|
end
|
104
109
|
|
105
110
|
it "starts new process imediatly when old exists" do
|
106
|
-
|
111
|
+
time_taken{
|
107
112
|
`ruby spec/cases/parallel_map_uneven.rb`
|
108
|
-
|
113
|
+
}.should <= 3.5
|
109
114
|
end
|
110
115
|
|
111
116
|
it "does not flatten results" do
|
@@ -124,12 +129,12 @@ describe Parallel do
|
|
124
129
|
`ruby spec/cases/map_with_nested_arrays_and_nil.rb`.should == '[nil, [2, 2], [[3], [3]]]'
|
125
130
|
end
|
126
131
|
|
127
|
-
it '
|
128
|
-
`ruby spec/cases/map_with_processes_and_exceptions.rb 2>&1`.should =~ /^\d{4}
|
132
|
+
it 'stops all workers when one fails in process' do
|
133
|
+
`ruby spec/cases/map_with_processes_and_exceptions.rb 2>&1`.should =~ /^\d{4} raised$/
|
129
134
|
end
|
130
135
|
|
131
|
-
it '
|
132
|
-
`ruby spec/cases/map_with_threads_and_exceptions.rb 2>&1`.should =~ /^\d{0,4}
|
136
|
+
it 'stops all workers when one fails in thread' do
|
137
|
+
`ruby spec/cases/map_with_threads_and_exceptions.rb 2>&1`.should =~ /^\d{0,4} raised$/
|
133
138
|
end
|
134
139
|
|
135
140
|
it "can run with 0 threads" do
|
@@ -154,12 +159,12 @@ describe Parallel do
|
|
154
159
|
|
155
160
|
it "can run with 0 threads" do
|
156
161
|
Thread.should_not_receive(:exclusive)
|
157
|
-
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
162
|
+
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
158
163
|
end
|
159
164
|
|
160
165
|
it "can run with 0 processes" do
|
161
166
|
Process.should_not_receive(:fork)
|
162
|
-
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
167
|
+
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
163
168
|
end
|
164
169
|
end
|
165
170
|
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 27
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 5
|
9
|
-
-
|
10
|
-
version: 0.5.
|
9
|
+
- 8
|
10
|
+
version: 0.5.8
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Michael Grosser
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-08-
|
18
|
+
date: 2011-08-18 00:00:00 +02:00
|
19
19
|
default_executable:
|
20
20
|
dependencies: []
|
21
21
|
|