parallel 0.4.6 → 0.5.0

data/Gemfile

@@ -0,0 +1,5 @@
+source 'http://rubygems.org'
+
+gem 'rake'
+gem 'rspec', '~>1'
+gem 'jeweler'

data/Gemfile.lock

@@ -0,0 +1,22 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    gemcutter (0.6.1)
+    git (1.2.5)
+    jeweler (1.4.0)
+      gemcutter (>= 0.1.0)
+      git (>= 1.2.5)
+      rubyforge (>= 2.0.0)
+    json_pure (1.4.6)
+    rake (0.8.7)
+    rspec (1.3.1)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler
+  rake
+  rspec (~> 1)

data/Readme.md

@@ -51,6 +51,7 @@ Authors
  - [Fred Wu](http://fredwu.me)
  - [mikezter](http://github.com/mikezter)
  - [Jeremy Durham](http://www.jeremydurham.com)
+ - [Nick Gauthier](http://www.ngauthier.com)
 
 [Michael Grosser](http://pragmatig.wordpress.com)  
 grosser.michael@gmail.com  

data/VERSION

@@ -1 +1 @@
-0.4.6
+0.5.0

data/lib/parallel.rb

@@ -1,8 +1,8 @@
 require 'thread' # to get Thread.exclusive
+require 'base64'
 
 class Parallel
   VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
-  SPLAT_BUG = *[] # fix for bug/feature http://redmine.ruby-lang.org/issues/show/2422
 
   def self.in_threads(options={:count => 2})
     count, options = extract_count_from_options(options)
@@ -23,12 +23,7 @@ class Parallel
   def self.in_processes(options = {}, &block)
     count, options = extract_count_from_options(options)
     count ||= processor_count
-    preserve_results = (options[:preserve_results] != false)
-
-    pipes, pids = fork_and_start_writing(count, :preserve_results => preserve_results, &block)
-    out = read_from_pipes(pipes)
-    pids.each { |pid| Process.wait(pid) }
-    out.map{|x| deserialize(x) } if preserve_results
+    map(0...count, options.merge(:in_processes => count), &block)
   end
 
   def self.each(array, options={}, &block)
@@ -40,7 +35,7 @@ class Parallel
     each(array, options.merge(:with_index => true), &block)
   end
 
-  def self.map(array, options = {})
+  def self.map(array, options = {}, &block)
     array = array.to_a if array.is_a?(Range)
 
     if options[:in_threads]
@@ -50,26 +45,26 @@ class Parallel
       method = :in_processes
       size = options[method] || processor_count
     end
-
-    # work in #{size} threads that use threads/processes
-    results = []
-    current = -1
-
-    in_threads(size) do
-      # as long as there are more items, work on one of them
-      loop do
-        index = Thread.exclusive{ current+=1 }
-        break if index >= array.size
-        results[index] = *send(method, options.merge(:count => 1)) do
-          args = [array[index]]
-          args << index if options[:with_index]
-          yield *args
+    size = [array.size, size].min
+
+    if method == :in_threads
+      # work in #{size} threads that use threads/processes
+      results = []
+      current = -1
+
+      in_threads(size) do
+        # as long as there are more items, work on one of them
+        loop do
+          index = Thread.exclusive{ current+=1 }
+          break if index >= array.size
+          results[index] = call_with_index(array, index, options, &block)
         end
       end
-    end
 
-    results = results.flatten(1) if SPLAT_BUG 
-    results
+      results
+    else
+      work_in_processes(array, options.merge(:count => size), &block)
+    end
   end
 
   def self.map_with_index(array, options={}, &block)
@@ -91,51 +86,117 @@ class Parallel
 
   private
 
-  # Collect results from pipes simultanously
-  # otherwise pipes get stuck when to much is written (buffer full)
-  def self.read_from_pipes(reads)
-    out = []
-    in_threads(reads.size) do |i|
-      out[i] = ''
-      while text = reads[i].gets
-        out[i] += text
+  def self.work_in_processes(items, options, &blk)
+    workers = Array.new(options[:count]).map{ worker(items, options, &blk) }
+    Parallel.kill_on_ctrl_c(workers.map{|worker| worker[:pid] })
+
+    current_index = -1
+
+    # give every worker something to do
+    workers.each do |worker|
+      write_to_pipe(worker[:write], current_index += 1)
+    end
+
+    # fetch results and hand out new work
+    listener_threads = []
+    result = Array.new(items.size)
+
+    workers.each do |worker|
+      listener_threads << Thread.new do
+        begin
+          while output = worker[:read].gets
+            # store output from worker
+            result_index, output = decode(output.chomp)
+            raise output.exception if ExceptionWrapper === output
+            result[result_index] = output
+
+            # give worker next item
+            next_index = Thread.exclusive{ current_index += 1 }
+            break if next_index >= items.size
+            write_to_pipe(worker[:write], next_index)
+          end
+        ensure
+          worker[:read].close
+          worker[:write].close
+        end
       end
-      reads[i].close
     end
-    out
+
+    wait_for_threads(listener_threads)
+
+    # if they go zombie, rather wait here to be able to debug
+    wait_for_processes(workers.map{|worker| worker[:pid] })
+
+    result
   end
 
-  # fork and start writing results into n pipes
-  def self.fork_and_start_writing(count, options, &block)
-    reads = []
-    pids = []
-    count.times do |i|
-      reads[i], write = IO.pipe
-      pids << do_in_new_process(i, options.merge(:write_to => (options[:preserve_results] ? write : nil)), &block)
-      write.close
+  def self.worker(items, options, &block)
+    # use less memory on REE
+    GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+
+    child_read, parent_write = IO.pipe
+    parent_read, child_write = IO.pipe
+
+    pid = Process.fork do
+      parent_write.close
+      parent_read.close
+
+      begin
+        while input = child_read.gets and input != "\n"
+          index = decode(input.chomp)
+          begin
+            result = Parallel.call_with_index(items, index, options, &block)
+            result = nil if options[:preserve_results] == false
+          rescue Exception => e
+            result = ExceptionWrapper.new(e)
+          end
+          write_to_pipe(child_write, [index, result])
+        end
+      rescue Interrupt
+        child_read.close
+        child_write.close
+      end
     end
-    kill_on_ctrl_c(pids)
-    [reads, pids]
+
+    child_read.close
+    child_write.close
+
+    {:read => parent_read, :write => parent_write, :pid => pid}
   end
 
-  def self.do_in_new_process(work_item, options)
-    # activate copy on write friendly GC of REE
-    GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
-    Process.fork do
-      result = yield(work_item)
-      serialize(result, options) if options[:write_to]
+  def self.write_to_pipe(pipe, item)
+    pipe.write(encode(item))
+  end
+
+  def self.wait_for_threads(threads)
+    threads.each do |t|
+      begin
+        t.join
+      rescue Interrupt
+        # thread died, do not stop other threads
+      end
+    end
+  end
+
+  def self.wait_for_processes(pids)
+    pids.each do |pid|
+      begin
+        Process.wait(pid)
+      rescue Interrupt
+        # process died
+      end
     end
   end
 
-  def self.serialize(something, options)
-    Marshal.dump(something, options[:write_to])
+  def self.encode(obj)
+    Base64.encode64(Marshal.dump(obj)).split("\n").join + "\n"
   end
 
-  def self.deserialize(something)
-    Marshal.load(something)
+  def self.decode(str)
+    Marshal.load(Base64.decode64(str))
   end
 
-  # options is either a Interger or a Hash with :count
+  # options is either a Integer or a Hash with :count
   def self.extract_count_from_options(options)
     if options.is_a?(Hash)
       count = options[:count]
@@ -146,21 +207,6 @@ class Parallel
     [count, options]
   end
 
-  # split an array into groups of size items
-  # (copied from ActiveSupport, to not require it)
-  def self.in_groups_of(array, size)
-    results = []
-    loop do
-      slice = array[(results.size * size)...((results.size+1) * size)]
-      if slice.nil? or slice.empty?
-        break
-      else
-        results << slice
-      end
-    end
-    results
-  end
-
   # kill all these processes (children) if user presses Ctrl+c
   def self.kill_on_ctrl_c(pids)
     Signal.trap :SIGINT do
@@ -169,4 +215,17 @@ class Parallel
       exit 1 # Quit with 'failed' signal
     end
   end
-end
+
+  def self.call_with_index(array, index, options, &block)
+    args = [array[index]]
+    args << index if options[:with_index]
+    block.call(*args)
+  end
+
+  class ExceptionWrapper
+    attr_reader :exception
+    def initialize(exception)
+      @exception = exception
+    end
+  end
+end

data/parallel.gemspec

@@ -5,19 +5,21 @@
 
 Gem::Specification.new do |s|
   s.name = %q{parallel}
-  s.version = "0.4.6"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Michael Grosser"]
-  s.date = %q{2010-10-06}
+  s.date = %q{2010-10-24}
   s.email = %q{grosser.michael@gmail.com}
   s.files = [
-    "Rakefile",
+    "Gemfile",
+     "Gemfile.lock",
+     "Rakefile",
      "Readme.md",
      "VERSION",
      "lib/parallel.rb",
      "parallel.gemspec",
-     "spec/cases/cloeses_processes_at_runtime.rb",
+     "spec/cases/closes_processes_at_runtime.rb",
      "spec/cases/each.rb",
      "spec/cases/each_with_index.rb",
      "spec/cases/map_with_index.rb",
@@ -42,7 +44,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/grosser/parallel}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Run any kind of code in parallel processes}
   s.test_files = [
     "spec/spec_helper.rb",
@@ -57,8 +59,8 @@ Gem::Specification.new do |s|
      "spec/cases/parallel_start_and_kill.rb",
      "spec/cases/parallel_map_uneven.rb",
      "spec/cases/parallel_map_sleeping.rb",
-     "spec/cases/cloeses_processes_at_runtime.rb",
      "spec/cases/parallel_with_detected_cpus.rb",
+     "spec/cases/closes_processes_at_runtime.rb",
      "spec/cases/each.rb",
      "spec/cases/map_with_nested_arrays_and_nil.rb",
      "spec/cases/map_with_index_empty.rb",
@@ -72,7 +74,7 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
     else
     end
   else

data/spec/cases/no_dump_with_each.rb

@@ -10,7 +10,12 @@ class NotDumpable
   end
 end
 
-Parallel.each([NotDumpable.new]) do |x|
-  print 'not dumpable'
-  x
+Parallel.each([1]) do
+  print 'no dump for result'
+  NotDumpable.new
+end
+
+Parallel.each([NotDumpable.new]) do
+  print 'no dump for each'
+  1
 end

data/spec/parallel_spec.rb

@@ -30,9 +30,10 @@ describe Parallel do
         end
         sleep 1
         running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) }
-        uid_index         = running_processes.detect{ |line| line.include?("UID") }.index("UID") + 1
-        parent            = running_processes.detect{ |line| line.grep(/(0|)0:00(:|.)00/).any? and line.include?("ruby") }[uid_index]
-        `kill -2 #{parent}` #simulates Ctrl+c
+        pid_index = running_processes.detect{ |line| line.include?("UID") }.index("UID") + 1
+        parent_pid = running_processes.detect{ |line| line.grep(/(0|)0:00(:|.)00/).any? and line.include?("ruby") }[pid_index]
+        `kill -2 #{parent_pid}` #simulates Ctrl+c
+        sleep 1
       }.should_not change{`ps`.split("\n").size}
       Time.now.should be_close(t, 3)
     end
@@ -44,8 +45,7 @@ describe Parallel do
     end
 
     it "raises when one of the processes raises" do
-      pending 'there is some kind of error, but not the original...'
-      `ruby spec/cases/parallel_raise.rb`.should == 'TEST'
+      `ruby spec/cases/parallel_raise.rb`.strip.should == 'TEST'
     end
 
     it 'can handle to high fork rate' do
@@ -53,7 +53,7 @@ describe Parallel do
     end
 
     it 'it does not leave processes behind while running' do
-      `ruby spec/cases/cloeses_processes_at_runtime.rb`.should == 'OK'
+      `ruby spec/cases/closes_processes_at_runtime.rb`.should == 'OK'
     end
   end
 
@@ -127,7 +127,7 @@ describe Parallel do
     end
 
     it "does not use marshal_dump" do
-      `ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'not dumpable'
+      `ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'no dump for resultno dump for each'
     end
   end
 
@@ -136,22 +136,4 @@ describe Parallel do
       `ruby spec/cases/each_with_index.rb 2>&1`.should == 'a0b1'
     end
   end
-
-  describe :in_groups_of do
-    it "works for empty" do
-      Parallel.send(:in_groups_of, [], 3).should == []
-    end
-
-    it "works for smaller then count" do
-      Parallel.send(:in_groups_of, [1,2], 3).should == [[1,2]]
-    end
-
-    it "works for count" do
-      Parallel.send(:in_groups_of, [1,2,3], 3).should == [[1,2,3]]
-    end
-
-    it "works for larger than count" do
-      Parallel.send(:in_groups_of, [1,2,3,4], 3).should == [[1,2,3],[4]]
-    end
-  end
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: parallel
 version: !ruby/object:Gem::Version 
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
-  - 6
-  version: 0.4.6
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Michael Grosser
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-06 00:00:00 +02:00
+date: 2010-10-24 00:00:00 +02:00
 default_executable: 
 dependencies: []
 
@@ -27,12 +28,14 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- Gemfile
+- Gemfile.lock
 - Rakefile
 - Readme.md
 - VERSION
 - lib/parallel.rb
 - parallel.gemspec
-- spec/cases/cloeses_processes_at_runtime.rb
+- spec/cases/closes_processes_at_runtime.rb
 - spec/cases/each.rb
 - spec/cases/each_with_index.rb
 - spec/cases/map_with_index.rb
@@ -63,23 +66,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Run any kind of code in parallel processes
@@ -96,8 +103,8 @@ test_files:
 - spec/cases/parallel_start_and_kill.rb
 - spec/cases/parallel_map_uneven.rb
 - spec/cases/parallel_map_sleeping.rb
-- spec/cases/cloeses_processes_at_runtime.rb
 - spec/cases/parallel_with_detected_cpus.rb
+- spec/cases/closes_processes_at_runtime.rb
 - spec/cases/each.rb
 - spec/cases/map_with_nested_arrays_and_nil.rb
 - spec/cases/map_with_index_empty.rb