parallel_run 0.4 → 0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +10 -0
- data/lib/parallel_run.rb +48 -20
- data/parallel_run.gemspec +1 -1
- data/tests/helper.rb +15 -0
- data/tests/retry_test.rb +35 -0
- data/tests/simple_test.rb +107 -0
- data/tests/worker_test.rb +159 -0
- metadata +6 -2
data/Rakefile
CHANGED
data/lib/parallel_run.rb
CHANGED
@@ -43,6 +43,11 @@ optparse = OptionParser.new do|opts|
|
|
43
43
|
$options[:workers] = c
|
44
44
|
end
|
45
45
|
|
46
|
+
$options[:retry] = 1
|
47
|
+
opts.on('--retry RETRY', Integer, "Number of retry for each command") do |c|
|
48
|
+
$options[:retry] = c
|
49
|
+
end
|
50
|
+
|
46
51
|
opts.on('-h', '--help', 'Display this screen' ) do
|
47
52
|
puts opts
|
48
53
|
puts "Values can be used directly in command or prefix directly by using %v."
|
@@ -74,45 +79,68 @@ def replace s, value, k
|
|
74
79
|
s = s.gsub(/%k/, k.to_s)
|
75
80
|
s
|
76
81
|
end
|
82
|
+
|
77
83
|
module Enumerable
|
78
84
|
|
79
|
-
def
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
85
|
+
def in_groups(number, fill_with = nil)
|
86
|
+
# size / number gives minor group size;
|
87
|
+
# size % number gives how many objects need extra accomodation;
|
88
|
+
# each group hold either division or division + 1 items.
|
89
|
+
division = size / number
|
90
|
+
modulo = size % number
|
91
|
+
|
92
|
+
# create a new array avoiding dup
|
93
|
+
groups = []
|
94
|
+
start = 0
|
95
|
+
|
96
|
+
number.times do |index|
|
97
|
+
length = division + (modulo > 0 && modulo > index ? 1 : 0)
|
98
|
+
padding = fill_with != false &&
|
99
|
+
modulo > 0 && length == division ? 1 : 0
|
100
|
+
groups << slice(start, length).concat([fill_with] * padding)
|
101
|
+
start += length
|
102
|
+
end
|
103
|
+
|
104
|
+
if block_given?
|
105
|
+
groups.each{|g| yield(g) }
|
106
|
+
else
|
107
|
+
groups
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def peach(workers = nil, &b)
|
112
|
+
workers ||= count
|
84
113
|
|
85
114
|
threads = []
|
86
|
-
|
115
|
+
in_groups(workers, false).each do |slice|
|
87
116
|
threads << Thread.new(slice) do |thread_slice|
|
88
|
-
|
117
|
+
thread_slice.each{|elt| yield elt}
|
89
118
|
end
|
90
119
|
end
|
91
120
|
threads.each{|t| t.join }
|
92
121
|
self
|
93
122
|
end
|
94
123
|
|
95
|
-
def peach(pool = nil, &b)
|
96
|
-
_peach_run(pool) do |thread_slice, idx, div|
|
97
|
-
thread_slice.each{|elt| yield elt}
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
124
|
end
|
102
125
|
|
103
126
|
result = {}
|
104
127
|
ARGV.each_with_index.to_a.peach($options[:workers]) do |f, index|
|
128
|
+
try = $options[:retry]
|
105
129
|
c = replace $options[:command], f, index
|
106
130
|
prefix = replace $options[:prefix], f, index
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
131
|
+
while try > 0
|
132
|
+
output("Lauching command for #{f} : #{c}", prefix) if $options[:verbose]
|
133
|
+
full_cmd = "#{$options[:shell]} -c \"#{c.gsub(/"/, "\\\"")}\""
|
134
|
+
status = POpen4::popen4(full_cmd, 'r') do |stdout, stderr, stdin, pid|
|
135
|
+
[stdout, stderr].peach do |io|
|
136
|
+
flow = io == stderr ? "err" : "out"
|
137
|
+
while !io.eof?
|
138
|
+
output io.readline, prefix, flow
|
139
|
+
end
|
114
140
|
end
|
115
141
|
end
|
142
|
+
break if status.exitstatus == 0
|
143
|
+
try -= 1
|
116
144
|
end
|
117
145
|
result[f] = {
|
118
146
|
:command => c,
|
data/parallel_run.gemspec
CHANGED
data/tests/helper.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
require 'tempfile'
|
3
|
+
|
4
|
+
def parallel_run command_line, script
|
5
|
+
f = Tempfile.new 'script'
|
6
|
+
f.write script
|
7
|
+
f.close
|
8
|
+
File.chmod(0700, f.path)
|
9
|
+
command_line = command_line.gsub(/%s/, f.path)
|
10
|
+
exec = File.join(File.dirname(__FILE__), '..', 'bin', 'parallel_run')
|
11
|
+
path = File.join(File.dirname(__FILE__), '..', 'lib')
|
12
|
+
result = %x{RUBYLIB=#{path} #{exec} #{command_line}}
|
13
|
+
code = $?.exitstatus
|
14
|
+
return code, result
|
15
|
+
end
|
data/tests/retry_test.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper.rb')
|
2
|
+
|
3
|
+
class RetrySimple < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def test_no_retry
|
6
|
+
%x{rm -f /tmp/toto}
|
7
|
+
code, result = parallel_run "--command \"%s %k\" 1 2 3 4", <<-EOF
|
8
|
+
#!/bin/sh
|
9
|
+
|
10
|
+
sleep $1
|
11
|
+
test -f /tmp/toto
|
12
|
+
result=$?
|
13
|
+
touch /tmp/toto
|
14
|
+
exit $result
|
15
|
+
EOF
|
16
|
+
assert_equal 1, code
|
17
|
+
%x{rm -f /tmp/toto}
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_retry
|
21
|
+
%x{rm -f /tmp/toto}
|
22
|
+
code, result = parallel_run "--retry 2 --command \"%s %k\" 1 2 3 4", <<-EOF
|
23
|
+
#!/bin/sh
|
24
|
+
|
25
|
+
sleep $1
|
26
|
+
test -f /tmp/toto
|
27
|
+
result=$?
|
28
|
+
touch /tmp/toto
|
29
|
+
exit $result
|
30
|
+
EOF
|
31
|
+
assert_equal 0, code
|
32
|
+
%x{rm -f /tmp/toto}
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper.rb')
|
2
|
+
|
3
|
+
class TestSimple < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def test_index
|
6
|
+
code, result = parallel_run "--command \"%s %k\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
|
7
|
+
#!/bin/sh
|
8
|
+
echo $1
|
9
|
+
EOF
|
10
|
+
assert_equal 0, code
|
11
|
+
[0, 1, 2, 3, 4].each do |x|
|
12
|
+
assert result.match(/#{x}$/), "Not found #{x}"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_value
|
17
|
+
code, result = parallel_run "--command \"%s %v\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
|
18
|
+
#!/bin/sh
|
19
|
+
echo $1
|
20
|
+
EOF
|
21
|
+
assert_equal 0, code
|
22
|
+
[0, 1, 2, 3, 4].each do |x|
|
23
|
+
assert result.match(/val#{x}\/#{x + 5}$/), "Not found #{x}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_dirname
|
28
|
+
code, result = parallel_run "--command \"%s %d\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
|
29
|
+
#!/bin/sh
|
30
|
+
echo $1
|
31
|
+
EOF
|
32
|
+
assert_equal 0, code
|
33
|
+
[0, 1, 2, 3, 4].each do |x|
|
34
|
+
assert result.match(/val#{x}$/), "Not found #{x}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_basename
|
39
|
+
code, result = parallel_run "--command \"%s %b\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
|
40
|
+
#!/bin/sh
|
41
|
+
echo $1
|
42
|
+
EOF
|
43
|
+
assert_equal 0, code
|
44
|
+
[0, 1, 2, 3, 4].each do |x|
|
45
|
+
assert result.match(/#{x + 5}$/), "Not found #{x}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_shell
|
50
|
+
code, result = parallel_run "--command \"%s\" 1", <<-EOF
|
51
|
+
echo $BASH
|
52
|
+
EOF
|
53
|
+
assert_equal 0, code
|
54
|
+
assert result.match(/#{ENV['SHELL']}/)
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_shell_sh
|
58
|
+
code, result = parallel_run "--shell /bin/sh --command \"%s\" 1", <<-EOF
|
59
|
+
echo $BASH
|
60
|
+
EOF
|
61
|
+
assert_equal 0, code
|
62
|
+
assert result.match(/\/bin\/sh/)
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_parallel
|
66
|
+
code, result = parallel_run "--command \"%s %k\" 1 2 3 4 5", <<-EOF
|
67
|
+
#!/bin/sh
|
68
|
+
echo Start $1
|
69
|
+
sleep 0.5
|
70
|
+
echo Stop $1
|
71
|
+
sleep 0.5
|
72
|
+
EOF
|
73
|
+
assert_equal 0, code
|
74
|
+
start = 0
|
75
|
+
stop = 0
|
76
|
+
result.split(/\n/).each do |x|
|
77
|
+
start += 1 if x.match(/Start/)
|
78
|
+
stop += 1 if x.match(/Stop/)
|
79
|
+
assert_equal 5, start if stop == 1
|
80
|
+
end
|
81
|
+
assert_equal 5, start
|
82
|
+
assert_equal 5, stop
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_one_fail
|
86
|
+
code, result = parallel_run "--command \"%s %k\" val1 val2 val3 val4 val5", <<-EOF
|
87
|
+
if [ "$1" = 2 ]; then
|
88
|
+
exit 2
|
89
|
+
fi
|
90
|
+
EOF
|
91
|
+
assert_equal 1, code
|
92
|
+
assert result.match(/Command failed : val3/)
|
93
|
+
assert ! result.match(/Command failed : val2/)
|
94
|
+
end
|
95
|
+
|
96
|
+
def test_all_fail
|
97
|
+
code, result = parallel_run "--command \"%s %k\" val1 val2 val3 val4 val5", <<-EOF
|
98
|
+
exit 2
|
99
|
+
EOF
|
100
|
+
assert_equal 1, code
|
101
|
+
[1, 2, 3, 4, 5].each do |x|
|
102
|
+
assert result.match(/Command failed : val#{x}/)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
@@ -0,0 +1,159 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper.rb')
|
2
|
+
|
3
|
+
class WorkerSimple < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def test_worker_two
|
6
|
+
code, result = parallel_run "--workers 2 --command \"%s %k\" 1 2 3 4", <<-EOF
|
7
|
+
#!/bin/sh
|
8
|
+
echo Start $1
|
9
|
+
sleep 0.5
|
10
|
+
echo Stop $1
|
11
|
+
sleep 0.5
|
12
|
+
EOF
|
13
|
+
assert_equal 0, code
|
14
|
+
start = 0
|
15
|
+
stop = 0
|
16
|
+
result.split(/\n/).each do |x|
|
17
|
+
start += 1 if x.match(/Start/)
|
18
|
+
if x.match(/Stop/)
|
19
|
+
stop += 1
|
20
|
+
assert_equal 2, start if stop == 2
|
21
|
+
end
|
22
|
+
end
|
23
|
+
assert_equal 4, start
|
24
|
+
assert_equal 4, stop
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_worker_three
|
28
|
+
code, result = parallel_run "--workers 3 --command \"%s %k\" 1 2 3 4", <<-EOF
|
29
|
+
#!/bin/sh
|
30
|
+
echo Start $1
|
31
|
+
sleep 0.5
|
32
|
+
echo Stop $1
|
33
|
+
sleep 0.5
|
34
|
+
EOF
|
35
|
+
assert_equal 0, code
|
36
|
+
start = 0
|
37
|
+
stop = 0
|
38
|
+
result.split(/\n/).each do |x|
|
39
|
+
start += 1 if x.match(/Start/)
|
40
|
+
if x.match(/Stop/)
|
41
|
+
stop += 1
|
42
|
+
assert_equal 3, start if stop == 3
|
43
|
+
end
|
44
|
+
end
|
45
|
+
assert_equal 4, start
|
46
|
+
assert_equal 4, stop
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_worker_one
|
50
|
+
code, result = parallel_run "--workers 1 --command \"%s %k\" 1 2 3 4", <<-EOF
|
51
|
+
#!/bin/sh
|
52
|
+
echo Start $1
|
53
|
+
sleep 0.5
|
54
|
+
echo Stop $1
|
55
|
+
sleep 0.5
|
56
|
+
EOF
|
57
|
+
assert_equal 0, code
|
58
|
+
start = 0
|
59
|
+
stop = 0
|
60
|
+
result.split(/\n/).each do |x|
|
61
|
+
start += 1 if x.match(/Start/)
|
62
|
+
if x.match(/Stop/)
|
63
|
+
stop += 1
|
64
|
+
assert_equal stop, start
|
65
|
+
end
|
66
|
+
end
|
67
|
+
assert_equal 4, start
|
68
|
+
assert_equal 4, stop
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_worker_impair_one
|
72
|
+
code, result = parallel_run "--workers 1 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
|
73
|
+
#!/bin/sh
|
74
|
+
echo Start $1
|
75
|
+
sleep 0.5
|
76
|
+
echo Stop $1
|
77
|
+
sleep 0.5
|
78
|
+
EOF
|
79
|
+
assert_equal 0, code
|
80
|
+
start = 0
|
81
|
+
stop = 0
|
82
|
+
result.split(/\n/).each do |x|
|
83
|
+
start += 1 if x.match(/Start/)
|
84
|
+
if x.match(/Stop/)
|
85
|
+
stop += 1
|
86
|
+
assert_equal stop, start
|
87
|
+
end
|
88
|
+
end
|
89
|
+
assert_equal 9, start
|
90
|
+
assert_equal 9, stop
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_worker_impair_two
|
94
|
+
code, result = parallel_run "--workers 2 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
|
95
|
+
#!/bin/sh
|
96
|
+
echo Start $1
|
97
|
+
sleep 0.5
|
98
|
+
echo Stop $1
|
99
|
+
sleep 0.5
|
100
|
+
EOF
|
101
|
+
assert_equal 0, code
|
102
|
+
start = 0
|
103
|
+
stop = 0
|
104
|
+
result.split(/\n/).each do |x|
|
105
|
+
start += 1 if x.match(/Start/)
|
106
|
+
if x.match(/Stop/)
|
107
|
+
stop += 1
|
108
|
+
assert_equal stop, start if stop % 2 == 0
|
109
|
+
end
|
110
|
+
end
|
111
|
+
assert_equal 9, start
|
112
|
+
assert_equal 9, stop
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_worker_impair_three
|
116
|
+
code, result = parallel_run "--workers 3 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
|
117
|
+
#!/bin/sh
|
118
|
+
echo Start $1
|
119
|
+
sleep 0.5
|
120
|
+
echo Stop $1
|
121
|
+
sleep 0.5
|
122
|
+
EOF
|
123
|
+
assert_equal 0, code
|
124
|
+
start = 0
|
125
|
+
stop = 0
|
126
|
+
result.split(/\n/).each do |x|
|
127
|
+
start += 1 if x.match(/Start/)
|
128
|
+
if x.match(/Stop/)
|
129
|
+
stop += 1
|
130
|
+
assert_equal stop, start if stop % 3 == 0
|
131
|
+
end
|
132
|
+
end
|
133
|
+
assert_equal 9, start
|
134
|
+
assert_equal 9, stop
|
135
|
+
end
|
136
|
+
|
137
|
+
def test_worker_impair_for
|
138
|
+
code, result = parallel_run "--workers 4 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
|
139
|
+
#!/bin/sh
|
140
|
+
echo Start $1
|
141
|
+
sleep 0.5
|
142
|
+
echo Stop $1
|
143
|
+
sleep 0.5
|
144
|
+
EOF
|
145
|
+
assert_equal 0, code
|
146
|
+
start = 0
|
147
|
+
stop = 0
|
148
|
+
result.split(/\n/).each do |x|
|
149
|
+
start += 1 if x.match(/Start/)
|
150
|
+
if x.match(/Stop/)
|
151
|
+
stop += 1
|
152
|
+
assert_equal stop, start if stop == 4
|
153
|
+
end
|
154
|
+
end
|
155
|
+
assert_equal 9, start
|
156
|
+
assert_equal 9, stop
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel_run
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.5'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-12-
|
12
|
+
date: 2013-12-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: popen4
|
@@ -44,6 +44,10 @@ files:
|
|
44
44
|
- bin/parallel_run
|
45
45
|
- lib/parallel_run.rb
|
46
46
|
- parallel_run.gemspec
|
47
|
+
- tests/helper.rb
|
48
|
+
- tests/retry_test.rb
|
49
|
+
- tests/simple_test.rb
|
50
|
+
- tests/worker_test.rb
|
47
51
|
homepage: http://github.com/bpaquet/parallel_run
|
48
52
|
licenses: []
|
49
53
|
post_install_message:
|