parallel_run 0.4 → 0.5

data/Rakefile

@@ -1,2 +1,12 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+
+require 'rake'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "tests/*_test.rb"
+  t.verbose = true
+end
+
+task :default => :test

data/lib/parallel_run.rb

@@ -43,6 +43,11 @@ optparse = OptionParser.new do|opts|
     $options[:workers] = c
   end
 
+  $options[:retry] = 1
+  opts.on('--retry RETRY', Integer, "Number of retry for each command") do |c|
+    $options[:retry] = c
+  end
+
   opts.on('-h', '--help', 'Display this screen' ) do
     puts opts
     puts "Values can be used directly in command or prefix directly by using %v."
@@ -74,45 +79,68 @@ def replace s, value, k
   s = s.gsub(/%k/, k.to_s)
   s
 end
+
 module Enumerable
 
-  def _peach_run(pool = nil, &b)
-    pool ||= count
-    pool = 1 unless pool >= 1
-    div = (count.to_f/pool.to_f).ceil # should already be integer
-    div = 1 unless div >= 1 # each thread better do something!
+  def in_groups(number, fill_with = nil)
+    # size / number gives minor group size;
+    # size % number gives how many objects need extra accomodation;
+    # each group hold either division or division + 1 items.
+    division = size / number
+    modulo = size % number
+
+    # create a new array avoiding dup
+    groups = []
+    start = 0
+
+    number.times do |index|
+      length = division + (modulo > 0 && modulo > index ? 1 : 0)
+      padding = fill_with != false &&
+        modulo > 0 && length == division ? 1 : 0
+      groups << slice(start, length).concat([fill_with] * padding)
+      start += length
+    end
+
+    if block_given?
+      groups.each{|g| yield(g) }
+    else
+      groups
+    end
+  end
+
+  def peach(workers = nil, &b)
+    workers ||= count
 
     threads = []
-    each_slice(div).with_index do |slice, idx|
+    in_groups(workers, false).each do |slice|
       threads << Thread.new(slice) do |thread_slice|
-        yield thread_slice, idx, div
+        thread_slice.each{|elt| yield elt}
       end
     end
     threads.each{|t| t.join }
     self
   end
 
-  def peach(pool = nil, &b)
-    _peach_run(pool) do |thread_slice, idx, div|
-      thread_slice.each{|elt| yield elt}
-    end
-  end
-
 end
 
 result = {}
 ARGV.each_with_index.to_a.peach($options[:workers]) do |f, index|
+  try = $options[:retry]
   c = replace $options[:command], f, index
   prefix = replace $options[:prefix], f, index
-  output("Lauching command for #{f} : #{c}", prefix) if $options[:verbose]
-  full_cmd = "#{$options[:shell]} -c \"#{c.gsub(/"/, "\\\"")}\""
-  status = POpen4::popen4(full_cmd, 'r') do |stdout, stderr, stdin, pid|
-    [stdout, stderr].peach do |io|
-      flow = io == stderr ? "err" : "out"
-      while !io.eof?
-        output io.readline, prefix, flow
+  while try > 0
+    output("Lauching command for #{f} : #{c}", prefix) if $options[:verbose]
+    full_cmd = "#{$options[:shell]} -c \"#{c.gsub(/"/, "\\\"")}\""
+    status = POpen4::popen4(full_cmd, 'r') do |stdout, stderr, stdin, pid|
+      [stdout, stderr].peach do |io|
+        flow = io == stderr ? "err" : "out"
+        while !io.eof?
+          output io.readline, prefix, flow
+        end
       end
     end
+    break if status.exitstatus == 0
+    try -= 1
   end
   result[f] = {
     :command => c,

data/parallel_run.gemspec

@@ -4,7 +4,7 @@ $:.push File.expand_path("../lib", __FILE__)
 Gem::Specification.new do |s|
 
   s.name        = "parallel_run"
-  s.version     = "0.4"
+  s.version     = "0.5"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Bertrand Paquet"]
   s.email       = ["bertrand.paquet@gmail.com"]

data/tests/helper.rb

@@ -0,0 +1,15 @@
+require 'test/unit'
+require 'tempfile'
+
+def parallel_run command_line, script
+	f = Tempfile.new 'script'
+	f.write script
+	f.close
+	File.chmod(0700, f.path)
+	command_line = command_line.gsub(/%s/, f.path)
+	exec = File.join(File.dirname(__FILE__), '..', 'bin', 'parallel_run')
+	path = File.join(File.dirname(__FILE__), '..', 'lib')
+	result = %x{RUBYLIB=#{path} #{exec} #{command_line}}
+	code = $?.exitstatus
+	return code, result
+end

data/tests/retry_test.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), 'helper.rb')
+
+class RetrySimple < Test::Unit::TestCase
+
+	def test_no_retry
+		%x{rm -f /tmp/toto}
+		code, result = parallel_run "--command \"%s %k\" 1 2 3 4", <<-EOF
+#!/bin/sh
+
+sleep $1
+test -f /tmp/toto
+result=$?
+touch /tmp/toto
+exit $result
+EOF
+		assert_equal 1, code
+		%x{rm -f /tmp/toto}
+	end
+
+	def test_retry
+		%x{rm -f /tmp/toto}
+		code, result = parallel_run "--retry 2 --command \"%s %k\" 1 2 3 4", <<-EOF
+#!/bin/sh
+
+sleep $1
+test -f /tmp/toto
+result=$?
+touch /tmp/toto
+exit $result
+EOF
+		assert_equal 0, code
+		%x{rm -f /tmp/toto}
+	end
+
+end

data/tests/simple_test.rb

@@ -0,0 +1,107 @@
+require File.join(File.dirname(__FILE__), 'helper.rb')
+
+class TestSimple < Test::Unit::TestCase
+
+	def test_index
+		code, result = parallel_run "--command \"%s %k\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
+#!/bin/sh
+echo $1
+EOF
+		assert_equal 0, code
+		[0, 1, 2, 3, 4].each do |x|
+			assert result.match(/#{x}$/), "Not found #{x}"
+		end
+	end
+
+	def test_value
+		code, result = parallel_run "--command \"%s %v\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
+#!/bin/sh
+echo $1
+EOF
+		assert_equal 0, code
+		[0, 1, 2, 3, 4].each do |x|
+			assert result.match(/val#{x}\/#{x + 5}$/), "Not found #{x}"
+		end
+	end
+
+	def test_dirname
+		code, result = parallel_run "--command \"%s %d\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
+#!/bin/sh
+echo $1
+EOF
+		assert_equal 0, code
+		[0, 1, 2, 3, 4].each do |x|
+			assert result.match(/val#{x}$/), "Not found #{x}"
+		end
+	end
+
+	def test_basename
+		code, result = parallel_run "--command \"%s %b\" val0/5 val1/6 val2/7 val3/8 val4/9", <<-EOF
+#!/bin/sh
+echo $1
+EOF
+		assert_equal 0, code
+		[0, 1, 2, 3, 4].each do |x|
+			assert result.match(/#{x + 5}$/), "Not found #{x}"
+		end
+	end
+
+	def test_shell
+		code, result = parallel_run "--command \"%s\" 1", <<-EOF
+echo $BASH
+EOF
+		assert_equal 0, code
+		assert result.match(/#{ENV['SHELL']}/)
+	end
+
+	def test_shell_sh
+		code, result = parallel_run "--shell /bin/sh --command \"%s\" 1", <<-EOF
+echo $BASH
+EOF
+		assert_equal 0, code
+		assert result.match(/\/bin\/sh/)
+	end
+
+	def test_parallel
+		code, result = parallel_run "--command \"%s %k\" 1 2 3 4 5", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			stop += 1 if x.match(/Stop/)
+			assert_equal 5, start if stop == 1
+		end
+		assert_equal 5, start
+		assert_equal 5, stop
+	end
+
+	def test_one_fail
+		code, result = parallel_run "--command \"%s %k\" val1 val2 val3 val4 val5", <<-EOF
+if [ "$1" = 2 ]; then
+	exit 2
+fi
+EOF
+		assert_equal 1, code
+		assert result.match(/Command failed : val3/)
+		assert ! result.match(/Command failed : val2/)
+	end
+
+	def test_all_fail
+		code, result = parallel_run "--command \"%s %k\" val1 val2 val3 val4 val5", <<-EOF
+exit 2
+EOF
+		assert_equal 1, code
+		[1, 2, 3, 4, 5].each do |x|
+			assert result.match(/Command failed : val#{x}/)
+		end
+	end
+
+end
+

data/tests/worker_test.rb

@@ -0,0 +1,159 @@
+require File.join(File.dirname(__FILE__), 'helper.rb')
+
+class WorkerSimple < Test::Unit::TestCase
+
+	def test_worker_two
+		code, result = parallel_run "--workers 2 --command \"%s %k\" 1 2 3 4", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal 2, start if stop == 2
+			end
+		end
+		assert_equal 4, start
+		assert_equal 4, stop
+	end
+
+	def test_worker_three
+		code, result = parallel_run "--workers 3 --command \"%s %k\" 1 2 3 4", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal 3, start if stop == 3
+			end
+		end
+		assert_equal 4, start
+		assert_equal 4, stop
+	end
+
+	def test_worker_one
+		code, result = parallel_run "--workers 1 --command \"%s %k\" 1 2 3 4", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal stop, start
+			end
+		end
+		assert_equal 4, start
+		assert_equal 4, stop
+	end
+
+	def test_worker_impair_one
+		code, result = parallel_run "--workers 1 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal stop, start
+			end
+		end
+		assert_equal 9, start
+		assert_equal 9, stop
+	end
+
+	def test_worker_impair_two
+		code, result = parallel_run "--workers 2 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal stop, start if stop % 2 == 0
+			end
+		end
+		assert_equal 9, start
+		assert_equal 9, stop
+	end
+
+	def test_worker_impair_three
+		code, result = parallel_run "--workers 3 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal stop, start if stop % 3 == 0
+			end
+		end
+		assert_equal 9, start
+		assert_equal 9, stop
+	end
+
+	def test_worker_impair_for
+		code, result = parallel_run "--workers 4 --command \"%s %k\" 1 2 3 4 5 6 7 8 9", <<-EOF
+#!/bin/sh
+echo Start $1
+sleep 0.5
+echo Stop $1
+sleep 0.5
+EOF
+		assert_equal 0, code
+		start = 0
+		stop = 0
+		result.split(/\n/).each do |x|
+			start += 1 if x.match(/Start/)
+			if x.match(/Stop/)
+				stop += 1
+				assert_equal stop, start if stop == 4
+			end
+		end
+		assert_equal 9, start
+		assert_equal 9, stop
+	end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel_run
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-06 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: popen4
@@ -44,6 +44,10 @@ files:
 - bin/parallel_run
 - lib/parallel_run.rb
 - parallel_run.gemspec
+- tests/helper.rb
+- tests/retry_test.rb
+- tests/simple_test.rb
+- tests/worker_test.rb
 homepage: http://github.com/bpaquet/parallel_run
 licenses: []
 post_install_message: