parallel 0.5.0 → 0.5.1

data/Gemfile

@@ -1,5 +1,5 @@
 source 'http://rubygems.org'
 
 gem 'rake'
-gem 'rspec', '~>1'
+gem 'rspec', '~>2'
 gem 'jeweler'

data/Gemfile.lock

@@ -1,6 +1,7 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    diff-lcs (1.1.2)
     gemcutter (0.6.1)
     git (1.2.5)
     jeweler (1.4.0)
@@ -9,7 +10,16 @@ GEM
       rubyforge (>= 2.0.0)
     json_pure (1.4.6)
     rake (0.8.7)
-    rspec (1.3.1)
+    rspec (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
+      rspec-mocks (~> 2.0.1)
+    rspec-core (2.0.1)
+    rspec-expectations (2.0.1)
+      diff-lcs (>= 1.1.2)
+    rspec-mocks (2.0.1)
+      rspec-core (~> 2.0.1)
+      rspec-expectations (~> 2.0.1)
     rubyforge (2.0.4)
       json_pure (>= 1.1.7)
 
@@ -19,4 +29,4 @@ PLATFORMS
 DEPENDENCIES
   jeweler
   rake
-  rspec (~> 1)
+  rspec (~> 2)

data/Rakefile

@@ -1,15 +1,16 @@
 task :default => :spec
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new {|t| t.spec_opts = ['--color'] }
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--backtrace --color'
+end
 
 begin
   require 'jeweler'
-  project_name = 'parallel'
   Jeweler::Tasks.new do |gem|
-    gem.name = project_name
+    gem.name = 'parallel'
     gem.summary = "Run any kind of code in parallel processes"
     gem.email = "grosser.michael@gmail.com"
-    gem.homepage = "http://github.com/grosser/#{project_name}"
+    gem.homepage = "http://github.com/grosser/#{gem.name}"
     gem.authors = ["Michael Grosser"]
   end
 

data/Readme.md

@@ -53,6 +53,6 @@ Authors
  - [Jeremy Durham](http://www.jeremydurham.com)
  - [Nick Gauthier](http://www.ngauthier.com)
 
-[Michael Grosser](http://pragmatig.wordpress.com)  
+[Michael Grosser](http://grosser.it)  
 grosser.michael@gmail.com  
 Hereby placed under public domain, do what you want, just do not hold me accountable...

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

data/lib/parallel.rb

@@ -16,7 +16,8 @@ class Parallel
       end
     end
 
-    threads.each{|t| t.join }
+    wait_for_threads(threads)
+
     out
   end
 
@@ -48,20 +49,7 @@ class Parallel
     size = [array.size, size].min
 
     if method == :in_threads
-      # work in #{size} threads that use threads/processes
-      results = []
-      current = -1
-
-      in_threads(size) do
-        # as long as there are more items, work on one of them
-        loop do
-          index = Thread.exclusive{ current+=1 }
-          break if index >= array.size
-          results[index] = call_with_index(array, index, options, &block)
-        end
-      end
-
-      results
+      work_in_threads(array, options.merge(:count => size), &block)
     else
       work_in_processes(array, options.merge(:count => size), &block)
     end
@@ -86,6 +74,33 @@ class Parallel
 
   private
 
+  def self.work_in_threads(items, options, &block)
+    results = []
+    current = -1
+    exception = nil
+
+    in_threads(options[:count]) do
+      # as long as there are more items, work on one of them
+      loop do
+        break if exception
+
+        index = Thread.exclusive{ current+=1 }
+        break if index >= items.size
+
+        begin
+          results[index] = call_with_index(items, index, options, &block)
+        rescue Exception => e
+          exception = e
+          break
+        end
+      end
+    end
+
+    raise exception if exception
+
+    results
+  end
+
   def self.work_in_processes(items, options, &blk)
     workers = Array.new(options[:count]).map{ worker(items, options, &blk) }
     Parallel.kill_on_ctrl_c(workers.map{|worker| worker[:pid] })
@@ -100,6 +115,7 @@ class Parallel
     # fetch results and hand out new work
     listener_threads = []
     result = Array.new(items.size)
+    exception = nil
 
     workers.each do |worker|
       listener_threads << Thread.new do
@@ -107,7 +123,13 @@ class Parallel
           while output = worker[:read].gets
             # store output from worker
             result_index, output = decode(output.chomp)
-            raise output.exception if ExceptionWrapper === output
+            if ExceptionWrapper === output
+              exception = output.exception
+              break
+            elsif exception # some other thread failed
+              break
+            end
+
             result[result_index] = output
 
             # give worker next item
@@ -127,6 +149,8 @@ class Parallel
     # if they go zombie, rather wait here to be able to debug
     wait_for_processes(workers.map{|worker| worker[:pid] })
 
+    raise exception if exception
+
     result
   end
 
@@ -138,21 +162,12 @@ class Parallel
     parent_read, child_write = IO.pipe
 
     pid = Process.fork do
-      parent_write.close
-      parent_read.close
-
       begin
-        while input = child_read.gets and input != "\n"
-          index = decode(input.chomp)
-          begin
-            result = Parallel.call_with_index(items, index, options, &block)
-            result = nil if options[:preserve_results] == false
-          rescue Exception => e
-            result = ExceptionWrapper.new(e)
-          end
-          write_to_pipe(child_write, [index, result])
-        end
-      rescue Interrupt
+        parent_write.close
+        parent_read.close
+
+        process_incoming_jobs(child_read, child_write, items, options, &block)
+      ensure
         child_read.close
         child_write.close
       end
@@ -164,12 +179,25 @@ class Parallel
     {:read => parent_read, :write => parent_write, :pid => pid}
   end
 
+  def self.process_incoming_jobs(read, write, items, options, &block)
+    while input = read.gets and input != "\n"
+      index = decode(input.chomp)
+      begin
+        result = call_with_index(items, index, options, &block)
+        result = nil if options[:preserve_results] == false
+      rescue Exception => e
+        result = ExceptionWrapper.new(e)
+      end
+      write_to_pipe(write, [index, result])
+    end
+  end
+
   def self.write_to_pipe(pipe, item)
     pipe.write(encode(item))
   end
 
   def self.wait_for_threads(threads)
-    threads.each do |t|
+    threads.compact.each do |t|
       begin
         t.join
       rescue Interrupt

data/parallel.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{parallel}
-  s.version = "0.5.0"
+  s.version = "0.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Michael Grosser"]
-  s.date = %q{2010-10-24}
+  s.date = %q{2010-11-05}
   s.email = %q{grosser.michael@gmail.com}
   s.files = [
     "Gemfile",
@@ -25,6 +25,8 @@ Gem::Specification.new do |s|
      "spec/cases/map_with_index.rb",
      "spec/cases/map_with_index_empty.rb",
      "spec/cases/map_with_nested_arrays_and_nil.rb",
+     "spec/cases/map_with_processes_and_exceptions.rb",
+     "spec/cases/map_with_threads_and_exceptions.rb",
      "spec/cases/no_dump_with_each.rb",
      "spec/cases/parallel_high_fork_rate.rb",
      "spec/cases/parallel_influence_outside_data.rb",
@@ -50,11 +52,13 @@ Gem::Specification.new do |s|
     "spec/spec_helper.rb",
      "spec/parallel_spec.rb",
      "spec/cases/parallel_sleeping_2.rb",
+     "spec/cases/map_with_processes_and_exceptions.rb",
      "spec/cases/no_dump_with_each.rb",
      "spec/cases/parallel_high_fork_rate.rb",
      "spec/cases/map_with_index.rb",
      "spec/cases/parallel_with_set_processes.rb",
      "spec/cases/parallel_map.rb",
+     "spec/cases/map_with_threads_and_exceptions.rb",
      "spec/cases/parallel_influence_outside_data.rb",
      "spec/cases/parallel_start_and_kill.rb",
      "spec/cases/parallel_map_uneven.rb",

data/spec/cases/map_with_processes_and_exceptions.rb

@@ -0,0 +1,16 @@
+require File.expand_path('spec/spec_helper')
+
+class Parallel
+  def self.wait_for_threads(threads)
+    print ' all joined'
+  end
+end
+
+begin
+  Parallel.map(1..100, :in_processes => 4) do |x|
+    print x
+    raise 'foo' if x == 1
+  end
+rescue
+  print ' raised'
+end

data/spec/cases/map_with_threads_and_exceptions.rb

@@ -0,0 +1,16 @@
+require File.expand_path('spec/spec_helper')
+
+class Parallel
+  def self.wait_for_threads(threads)
+    print ' all joined'
+  end
+end
+
+begin
+  Parallel.map(1..100, :in_threads => 4) do |x|
+    print x
+    raise 'foo' if x == 1
+  end
+rescue
+  print ' raised'
+end

data/spec/parallel_spec.rb

@@ -2,16 +2,16 @@ require File.expand_path('spec/spec_helper')
 
 describe Parallel do
   describe :in_processes do
-    before do
-      @cpus = Parallel.processor_count
+    def cpus
+      Parallel.processor_count
     end
 
     it "executes with detected cpus" do
-      `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * @cpus
+      `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * cpus
     end
 
     it "executes with detected cpus when nil was given" do
-      `ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * @cpus
+      `ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * cpus
     end
 
     it "set amount of parallel processes" do
@@ -52,7 +52,7 @@ describe Parallel do
       `ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
     end
 
-    it 'it does not leave processes behind while running' do
+    it 'does not leave processes behind while running' do
       `ruby spec/cases/closes_processes_at_runtime.rb`.should == 'OK'
     end
   end
@@ -109,6 +109,14 @@ describe Parallel do
     it 'handles nested arrays and nil correctly' do
       `ruby spec/cases/map_with_nested_arrays_and_nil.rb`.should == '[nil, [2, 2], [[3], [3]]]'
     end
+
+    it 'joins all workers, when one fails in process' do
+      `ruby spec/cases/map_with_processes_and_exceptions.rb 2>&1`.should =~ /^\d{4} all joined raised$/
+    end
+
+    it 'joins all workers, when one fails in thread' do
+      `ruby spec/cases/map_with_threads_and_exceptions.rb 2>&1`.should =~ /^\d{0,4} all joined raised$/
+    end
   end
 
   describe :map_with_index do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: parallel
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 9
   prerelease: false
   segments: 
   - 0
   - 5
-  - 0
-  version: 0.5.0
+  - 1
+  version: 0.5.1
 platform: ruby
 authors: 
 - Michael Grosser
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-24 00:00:00 +02:00
+date: 2010-11-05 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -41,6 +41,8 @@ files:
 - spec/cases/map_with_index.rb
 - spec/cases/map_with_index_empty.rb
 - spec/cases/map_with_nested_arrays_and_nil.rb
+- spec/cases/map_with_processes_and_exceptions.rb
+- spec/cases/map_with_threads_and_exceptions.rb
 - spec/cases/no_dump_with_each.rb
 - spec/cases/parallel_high_fork_rate.rb
 - spec/cases/parallel_influence_outside_data.rb
@@ -94,11 +96,13 @@ test_files:
 - spec/spec_helper.rb
 - spec/parallel_spec.rb
 - spec/cases/parallel_sleeping_2.rb
+- spec/cases/map_with_processes_and_exceptions.rb
 - spec/cases/no_dump_with_each.rb
 - spec/cases/parallel_high_fork_rate.rb
 - spec/cases/map_with_index.rb
 - spec/cases/parallel_with_set_processes.rb
 - spec/cases/parallel_map.rb
+- spec/cases/map_with_threads_and_exceptions.rb
 - spec/cases/parallel_influence_outside_data.rb
 - spec/cases/parallel_start_and_kill.rb
 - spec/cases/parallel_map_uneven.rb