parallel 0.5.21 → 0.6.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parallel (0.5.21)
+    parallel (0.6.0)
 
 GEM
   remote: http://rubygems.org/

data/lib/parallel.rb

@@ -3,308 +3,329 @@ require 'rbconfig'
 require 'parallel/version'
 
 module Parallel
-  def self.in_threads(options={:count => 2})
-    count, options = extract_count_from_options(options)
-
-    out = []
-    threads = []
-
-    count.times do |i|
-      threads[i] = Thread.new do
-        out[i] = yield(i)
-      end
-    end
-
-    wait_for_threads(threads)
-
-    out
+  class DeadWorker < EOFError
   end
 
-  def self.in_processes(options = {}, &block)
-    count, options = extract_count_from_options(options)
-    count ||= processor_count
-    map(0...count, options.merge(:in_processes => count), &block)
-  end
+  class ExceptionWrapper
+    attr_reader :exception
+    def initialize(exception)
+      dumpable = Marshal.dump(exception) rescue nil
+      unless dumpable
+        exception = RuntimeError.new("Undumpable Exception -- #{exception.inspect}")
+      end
 
-  def self.each(array, options={}, &block)
-    map(array, options.merge(:preserve_results => false), &block)
-    array
+      @exception = exception
+    end
   end
 
-  def self.each_with_index(array, options={}, &block)
-    each(array, options.merge(:with_index => true), &block)
-  end
+  class Worker
+    attr_reader :pid, :read, :write
+    def initialize(read, write, pid)
+      @read, @write, @pid = read, write, pid
+    end
 
-  def self.map(array, options = {}, &block)
-    array = array.to_a # turn Range and other Enumerable-s into an Array
+    def close_pipes
+      read.close
+      write.close
+    end
 
-    if options[:in_threads]
-      method = :in_threads
-      size = options[method]
-    else
-      method = :in_processes
-      size = options[method] || processor_count
+    def wait
+      Process.wait(pid)
+    rescue Interrupt
+      # process died
     end
-    size = [array.size, size].min
 
-    return work_direct(array, options, &block) if size == 0
+    def work(index)
+      begin
+        Marshal.dump(index, write)
+      rescue Errno::EPIPE
+        raise DeadWorker
+      end
 
-    if method == :in_threads
-      work_in_threads(array, options.merge(:count => size), &block)
-    else
-      work_in_processes(array, options.merge(:count => size), &block)
+      begin
+        Marshal.load(read)
+      rescue EOFError
+        raise Parallel::DeadWorker
+      end
     end
   end
 
-  def self.map_with_index(array, options={}, &block)
-    map(array, options.merge(:with_index => true), &block)
-  end
+  class << self
+    def in_threads(options={:count => 2})
+      count, options = extract_count_from_options(options)
 
-  def self.processor_count
-    @processor_count ||= case RbConfig::CONFIG['host_os']
-    when /darwin9/
-      `hwprefs cpu_count`.to_i
-    when /darwin/
-      (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
-    when /linux|cygwin/
-      `grep -c processor /proc/cpuinfo`.to_i
-    when /(open|free)bsd/
-      `sysctl -n hw.ncpu`.to_i
-    when /mswin|mingw/
-      require 'win32ole'
-      wmi = WIN32OLE.connect("winmgmts://")
-      cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
-      cpu.to_enum.first.NumberOfLogicalProcessors
-    when /solaris2/
-      `psrinfo -p`.to_i # this is physical cpus afaik
-    else
-      $stderr.puts "Unknown architecture ( #{RbConfig::CONFIG["host_os"]} ) assuming one processor."
-      1
+      out = []
+      threads = []
+
+      count.times do |i|
+        threads[i] = Thread.new do
+          out[i] = yield(i)
+        end
+      end
+
+      wait_for_threads(threads)
+
+      out
     end
-  end
 
-  def self.physical_processor_count
-    @physical_processor_count ||= case RbConfig::CONFIG['host_os']
-    when /darwin1/, /freebsd/
-      `sysctl -n hw.physicalcpu`.to_i
-    when /linux/
-      `grep cores /proc/cpuinfo`[/\d+/].to_i
-    when /mswin|mingw/
-      require 'win32ole'
-      wmi = WIN32OLE.connect("winmgmts://")
-      cpu = wmi.ExecQuery("select NumberOfProcessors from Win32_Processor")
-      cpu.to_enum.first.NumberOfLogicalProcessors
-    else
-      processor_count
+    def in_processes(options = {}, &block)
+      count, options = extract_count_from_options(options)
+      count ||= processor_count
+      map(0...count, options.merge(:in_processes => count), &block)
     end
-  end
 
-  private
+    def each(array, options={}, &block)
+      map(array, options.merge(:preserve_results => false), &block)
+      array
+    end
 
-  def self.work_direct(array, options)
-    results = []
-    array.each_with_index do |e,i|
-      results << (options[:with_index] ? yield(e,i) : yield(e))
+    def each_with_index(array, options={}, &block)
+      each(array, options.merge(:with_index => true), &block)
     end
-    results
-  end
 
-  def self.hwprefs_available?
-    `which hwprefs` != ''
-  end
+    def map(array, options = {}, &block)
+      array = array.to_a # turn Range and other Enumerable-s into an Array
 
-  def self.work_in_threads(items, options, &block)
-    results = []
-    current = -1
-    exception = nil
-    on_start = options[:start]
-    on_finish = options[:finish]
+      if options[:in_threads]
+        method = :in_threads
+        size = options[method]
+      else
+        method = :in_processes
+        size = options[method] || processor_count
+      end
+      size = [array.size, size].min
 
-    in_threads(options[:count]) do
-      # as long as there are more items, work on one of them
-      loop do
-        break if exception
+      return work_direct(array, options, &block) if size == 0
 
-        index = Thread.exclusive{ current+=1 }
-        break if index >= items.size
+      if method == :in_threads
+        work_in_threads(array, options.merge(:count => size), &block)
+      else
+        work_in_processes(array, options.merge(:count => size), &block)
+      end
+    end
 
-        item = items[index]
-        on_start.call(item, index) if on_start
+    def map_with_index(array, options={}, &block)
+      map(array, options.merge(:with_index => true), &block)
+    end
 
-        begin
-          results[index] = call_with_index(items, index, options, &block)
-        rescue Exception => e
-          exception = e
-          break
-        ensure
-          on_finish.call(item, index) if on_finish
-        end
+    def processor_count
+      @processor_count ||= case RbConfig::CONFIG['host_os']
+      when /darwin9/
+        `hwprefs cpu_count`.to_i
+      when /darwin/
+        (hwprefs_available? ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
+      when /linux|cygwin/
+        `grep -c processor /proc/cpuinfo`.to_i
+      when /(open|free)bsd/
+        `sysctl -n hw.ncpu`.to_i
+      when /mswin|mingw/
+        require 'win32ole'
+        wmi = WIN32OLE.connect("winmgmts://")
+        cpu = wmi.ExecQuery("select NumberOfLogicalProcessors from Win32_Processor")
+        cpu.to_enum.first.NumberOfLogicalProcessors
+      when /solaris2/
+        `psrinfo -p`.to_i # this is physical cpus afaik
+      else
+        $stderr.puts "Unknown architecture ( #{RbConfig::CONFIG["host_os"]} ) assuming one processor."
+        1
       end
     end
 
-    raise exception if exception
+    def physical_processor_count
+      @physical_processor_count ||= case RbConfig::CONFIG['host_os']
+      when /darwin1/, /freebsd/
+        `sysctl -n hw.physicalcpu`.to_i
+      when /linux/
+        `grep cores /proc/cpuinfo`[/\d+/].to_i
+      when /mswin|mingw/
+        require 'win32ole'
+        wmi = WIN32OLE.connect("winmgmts://")
+        cpu = wmi.ExecQuery("select NumberOfProcessors from Win32_Processor")
+        cpu.to_enum.first.NumberOfLogicalProcessors
+      else
+        processor_count
+      end
+    end
 
-    results
-  end
+    private
 
-  def self.work_in_processes(items, options, &blk)
-    workers = create_workers(items, options, &blk)
-    current_index = -1
-    results = []
-    exception = nil
-    on_start = options[:start]
-    on_finish = options[:finish]
+    def work_direct(array, options)
+      results = []
+      array.each_with_index do |e,i|
+        results << (options[:with_index] ? yield(e,i) : yield(e))
+      end
+      results
+    end
+
+    def hwprefs_available?
+      `which hwprefs` != ''
+    end
 
-    in_threads(options[:count]) do |i|
-      worker = workers[i]
+    def work_in_threads(items, options, &block)
+      results = []
+      current = -1
+      exception = nil
 
-      begin
+      in_threads(options[:count]) do
+        # as long as there are more items, work on one of them
         loop do
           break if exception
-          index = Thread.exclusive{ current_index += 1 }
-          break if index >= items.size
 
-          item = items[index]
-
-          Marshal.dump(index, worker[:write])
-          on_start.call(item, index) if on_start
-
-          output = Marshal.load(worker[:read])
-          on_finish.call(item, index) if on_finish
+          index = Thread.exclusive{ current+=1 }
+          break if index >= items.size
 
-          if ExceptionWrapper === output
-            exception = output.exception
-          else
-            results[index] = output
+          with_instrumentation items[index], index, options do
+            begin
+              results[index] = call_with_index(items, index, options, &block)
+            rescue Exception => e
+              exception = e
+              break
+            end
           end
         end
-      ensure
-        close_pipes(worker)
-        wait_for_process worker[:pid] # if it goes zombie, rather wait here to be able to debug
       end
-    end
 
-    raise exception if exception
+      raise exception if exception
 
-    results
-  end
-
-  def self.create_workers(items, options, &block)
-    workers = []
-    Array.new(options[:count]).each do
-      workers << worker(items, options.merge(:started_workers => workers), &block)
+      results
     end
 
-    pids = workers.map{|worker| worker[:pid] }
-    kill_on_ctrl_c(pids)
-    workers
-  end
+    def work_in_processes(items, options, &blk)
+      workers = create_workers(items, options, &blk)
+      current_index = -1
+      results = []
+      exception = nil
 
-  def self.worker(items, options, &block)
-    # use less memory on REE
-    GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+      in_threads(options[:count]) do |i|
+        worker = workers[i]
 
-    child_read, parent_write = IO.pipe
-    parent_read, child_write = IO.pipe
+        begin
+          loop do
+            break if exception
+            index = Thread.exclusive{ current_index += 1 }
+            break if index >= items.size
+
+            output = with_instrumentation items[index], index, options do
+              worker.work(index)
+            end
+
+            if ExceptionWrapper === output
+              exception = output.exception
+            else
+              results[index] = output
+            end
+          end
+        ensure
+          worker.close_pipes
+          worker.wait # if it goes zombie, rather wait here to be able to debug
+        end
+      end
 
-    pid = Process.fork do
-      begin
-        options.delete(:started_workers).each{|w| close_pipes w }
+      raise exception if exception
 
-        parent_write.close
-        parent_read.close
+      results
+    end
 
-        process_incoming_jobs(child_read, child_write, items, options, &block)
-      ensure
-        child_read.close
-        child_write.close
+    def create_workers(items, options, &block)
+      workers = []
+      Array.new(options[:count]).each do
+        workers << worker(items, options.merge(:started_workers => workers), &block)
       end
+
+      kill_on_ctrl_c(workers.map(&:pid))
+      workers
     end
 
-    child_read.close
-    child_write.close
+    def worker(items, options, &block)
+      # use less memory on REE
+      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
 
-    {:read => parent_read, :write => parent_write, :pid => pid}
-  end
+      child_read, parent_write = IO.pipe
+      parent_read, child_write = IO.pipe
 
-  def self.close_pipes(worker)
-    worker[:read].close
-    worker[:write].close
-  end
+      pid = Process.fork do
+        begin
+          options.delete(:started_workers).each(&:close_pipes)
 
-  def self.process_incoming_jobs(read, write, items, options, &block)
-    while !read.eof?
-      index = Marshal.load(read)
-      begin
-        result = call_with_index(items, index, options, &block)
-        result = nil if options[:preserve_results] == false
-      rescue Exception => e
-        result = ExceptionWrapper.new(e)
-      end
-      Marshal.dump(result, write)
-    end
-  end
+          parent_write.close
+          parent_read.close
 
-  def self.wait_for_threads(threads)
-    threads.compact.each do |t|
-      begin
-        t.join
-      rescue Interrupt
-        # thread died, do not stop other threads
+          process_incoming_jobs(child_read, child_write, items, options, &block)
+        ensure
+          child_read.close
+          child_write.close
+        end
       end
-    end
-  end
 
-  def self.wait_for_process(pid)
-    begin
-      Process.wait(pid)
-    rescue Interrupt
-      # process died
+      child_read.close
+      child_write.close
+
+      Worker.new(parent_read, parent_write, pid)
     end
-  end
 
-  # options is either a Integer or a Hash with :count
-  def self.extract_count_from_options(options)
-    if options.is_a?(Hash)
-      count = options[:count]
-    else
-      count = options
-      options = {}
+    def process_incoming_jobs(read, write, items, options, &block)
+      while !read.eof?
+        index = Marshal.load(read)
+        begin
+          result = call_with_index(items, index, options, &block)
+          result = nil if options[:preserve_results] == false
+        rescue Exception => e
+          result = ExceptionWrapper.new(e)
+        end
+        Marshal.dump(result, write)
+      end
     end
-    [count, options]
-  end
 
-  # kill all these processes (children) if user presses Ctrl+c
-  def self.kill_on_ctrl_c(pids)
-    Signal.trap :SIGINT do
-      $stderr.puts 'Parallel execution interrupted, exiting ...'
-      pids.compact.each do |pid|
+    def wait_for_threads(threads)
+      threads.compact.each do |t|
         begin
-          Process.kill(:KILL, pid)
-        rescue Errno::ESRCH
-          # some linux systems already automatically killed the children at this point
-          # so we just ignore them not being there
+          t.join
+        rescue Interrupt
+          # thread died, do not stop other threads
         end
       end
-      exit 1 # Quit with 'failed' signal
     end
-  end
 
-  def self.call_with_index(array, index, options, &block)
-    args = [array[index]]
-    args << index if options[:with_index]
-    block.call(*args)
-  end
+    # options is either a Integer or a Hash with :count
+    def extract_count_from_options(options)
+      if options.is_a?(Hash)
+        count = options[:count]
+      else
+        count = options
+        options = {}
+      end
+      [count, options]
+    end
 
-  class ExceptionWrapper
-    attr_reader :exception
-    def initialize(exception)
-      dumpable = Marshal.dump(exception) rescue nil
-      unless dumpable
-        exception = RuntimeError.new("Undumpable Exception -- #{exception.inspect}")
+    # kill all these processes (children) if user presses Ctrl+c
+    def kill_on_ctrl_c(pids)
+      Signal.trap :SIGINT do
+        $stderr.puts 'Parallel execution interrupted, exiting ...'
+        pids.compact.each do |pid|
+          begin
+            Process.kill(:KILL, pid)
+          rescue Errno::ESRCH
+            # some linux systems already automatically killed the children at this point
+            # so we just ignore them not being there
+          end
+        end
+        exit 1 # Quit with 'failed' signal
       end
+    end
 
-      @exception = exception
+    def call_with_index(array, index, options, &block)
+      args = [array[index]]
+      args << index if options[:with_index]
+      block.call(*args)
+    end
+
+    def with_instrumentation(item, index, options)
+      on_start = options[:start]
+      on_finish = options[:finish]
+      on_start.call(item, index) if on_start
+      yield
+    ensure
+      on_finish.call(item, index) if on_finish
     end
   end
 end

data/lib/parallel/version.rb

@@ -1,3 +1,3 @@
 module Parallel
-  VERSION = Version = '0.5.21'
+  VERSION = Version = '0.6.0'
 end

data/spec/cases/map_with_killed_worker_before_read.rb

@@ -0,0 +1,9 @@
+require File.expand_path('spec/spec_helper')
+
+begin
+  Parallel.map([1,2,3]) do |x, i|
+    Process.kill("SIGKILL", Process.pid)
+  end
+rescue Parallel::DeadWorker
+  puts "DEAD"
+end

data/spec/cases/map_with_killed_worker_before_write.rb

@@ -0,0 +1,18 @@
+require File.expand_path('spec/spec_helper')
+
+Parallel::Worker.class_eval do
+  alias_method :work_without_kill, :work
+  def work(*args)
+    Process.kill("SIGKILL", pid)
+    sleep 0.5
+    work_without_kill(*args)
+  end
+end
+
+begin
+  Parallel.map([1,2,3]) do |x, i|
+    Process.kill("SIGKILL", Process.pid)
+  end
+rescue Parallel::DeadWorker
+  puts "DEAD"
+end

data/spec/parallel_spec.rb

@@ -206,6 +206,14 @@ describe Parallel do
       monitor.should_receive(:call).once.with(:third, 2)
       Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 3) {}
     end
+
+    it "spits out a useful error when a worker dies before read" do
+      `ruby spec/cases/map_with_killed_worker_before_read.rb 2>&1`.should include "DEAD"
+    end
+
+    it "spits out a useful error when a worker dies before write" do
+      `ruby spec/cases/map_with_killed_worker_before_write.rb 2>&1`.should include "DEAD"
+    end
   end
 
   describe ".map_with_index" do

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: parallel
 version: !ruby/object:Gem::Version
-  version: 0.5.21
   prerelease: 
+  version: 0.6.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-01 00:00:00.000000000 Z
+date: 2012-12-15 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: michael@grosser.it
@@ -31,6 +31,8 @@ files:
 - spec/cases/host_os_override_processor_count.rb
 - spec/cases/map_with_index.rb
 - spec/cases/map_with_index_empty.rb
+- spec/cases/map_with_killed_worker_before_read.rb
+- spec/cases/map_with_killed_worker_before_write.rb
 - spec/cases/map_with_nested_arrays_and_nil.rb
 - spec/cases/map_with_processes_and_exceptions.rb
 - spec/cases/map_with_threads_and_exceptions.rb
@@ -59,23 +61,23 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
       segments:
       - 0
-      hash: -4018217091904548173
-required_rubygems_version: !ruby/object:Gem::Requirement
+      hash: -1821301885117501782
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
       segments:
       - 0
-      hash: -4018217091904548173
+      hash: -1821301885117501782
+  none: false
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24