parallel 0.3.7 → 0.4.0

data/VERSION

@@ -1 +1 @@
-0.3.7
+0.4.0

data/lib/parallel.rb

@@ -3,7 +3,9 @@ require 'thread' # to get Thread.exclusive
 class Parallel
   VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
 
-  def self.in_threads(count = 2)
+  def self.in_threads(options={:count => 2})
+    count, options = extract_count_from_options(options)
+
     out = []
     threads = []
 
@@ -17,40 +19,18 @@ class Parallel
     out
   end
 
-  def self.in_processes(count = nil)
+  def self.in_processes(options = {}, &block)
+    count, options = extract_count_from_options(options)
     count ||= processor_count
+    preserve_results = (options[:preserve_results] != false)
 
-    # Start writing results into n pipes
-    reads = []
-    writes = []
-    pids = []
-    count.times do |i|
-      reads[i], writes[i] = IO.pipe
-      # activate copy on write friendly GC of REE
-      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
-      pids << Process.fork do
-        Marshal.dump(yield(i), writes[i]) # Serialize result
-      end
-    end
-
-    kill_on_ctrl_c(pids)
-
-    # Collect results from pipes simultanously
-    # otherwise pipes get stuck when to much is written (buffer full)
-    out = []
-    in_threads(count) do |i|
-      writes[i].close
-      while text = reads[i].gets
-        out[i] = out[i].to_s + text
-      end
-      reads[i].close
-    end
-
-    out.map{|x| Marshal.load(x) } # Deserialize results
+    pipes = fork_and_start_writing(count, :preserve_results => preserve_results, &block)
+    out = read_from_pipes(pipes)
+    out.map{|x| deserialize(x) } if preserve_results
   end
 
   def self.each(array, options={}, &block)
-    map(array, options, &block)
+    map(array, options.merge(:preserve_results => false), &block)
     array
   end
 
@@ -74,7 +54,7 @@ class Parallel
       loop do
         index = Thread.exclusive{ current+=1 }
         break if index >= array.size
-        results[index] = *send(method, 1){ yield array[index] }
+        results[index] = *send(method, options.merge(:count => 1)){ yield array[index] }
       end
     end
 
@@ -92,7 +72,63 @@ class Parallel
 
   private
 
+  # Collect results from pipes simultanously
+  # otherwise pipes get stuck when to much is written (buffer full)
+  def self.read_from_pipes(reads)
+    out = []
+    in_threads(reads.size) do |i|
+      out[i] = ''
+      while text = reads[i].gets
+        out[i] += text
+      end
+      reads[i].close
+    end
+    out
+  end
+
+  # fork and start writing results into n pipes
+  def self.fork_and_start_writing(count, options, &block)
+    reads = []
+    pids = []
+    count.times do |i|
+      reads[i], write = IO.pipe
+      pids << do_in_new_process(i, options.merge(:write_to => (options[:preserve_results] ? write : nil)), &block)
+      write.close
+    end
+    kill_on_ctrl_c(pids)
+    reads
+  end
+
+  def self.do_in_new_process(work_item, options)
+    # activate copy on write friendly GC of REE
+    GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+    Process.fork do
+      result = yield(work_item)
+      serialize(result, options) if options[:write_to]
+    end
+  end
+
+  def self.serialize(something, options)
+    Marshal.dump(something, options[:write_to])
+  end
+
+  def self.deserialize(something)
+    Marshal.load(something)
+  end
+
+  # options is either a Interger or a Hash with :count
+  def self.extract_count_from_options(options)
+    if options.is_a?(Hash)
+      count = options[:count]
+    else
+      count = options
+      options = {}
+    end
+    [count, options]
+  end
+
   # split an array into groups of size items
+  # (copied from ActiveSupport, to not require it)
   def self.in_groups_of(array, size)
     results = []
     loop do
@@ -106,7 +142,7 @@ class Parallel
     results
   end
 
-  #handle user interrupt (Ctrl+c)
+  # kill all these processes (children) if user presses Ctrl+c
   def self.kill_on_ctrl_c(pids)
     Signal.trap :SIGINT do
       $stderr.puts 'Parallel execution interrupted, exiting ...'

data/parallel.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{parallel}
-  s.version = "0.3.7"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Michael Grosser"]
-  s.date = %q{2009-12-19}
+  s.date = %q{2010-01-17}
   s.email = %q{grosser.michael@gmail.com}
   s.extra_rdoc_files = [
     "README.markdown"
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
      "VERSION",
      "lib/parallel.rb",
      "parallel.gemspec",
+     "spec/cases/no_dump_with_each.rb",
      "spec/cases/parallel_each.rb",
      "spec/cases/parallel_high_fork_rate.rb",
      "spec/cases/parallel_influence_outside_data.rb",
@@ -47,6 +48,7 @@ Gem::Specification.new do |s|
      "spec/cases/parallel_map_uneven.rb",
      "spec/cases/parallel_map_range.rb",
      "spec/cases/parallel_with_set_processes.rb",
+     "spec/cases/no_dump_with_each.rb",
      "spec/cases/parallel_start_and_kill.rb",
      "spec/cases/parallel_raise.rb",
      "spec/cases/parallel_sleeping_2.rb",

data/spec/cases/no_dump_with_each.rb

@@ -0,0 +1,16 @@
+require 'spec/spec_helper.rb'
+
+class NotDumpable
+  def marshal_dump
+    raise "NOOOO"
+  end
+
+  def to_s
+    'not dumpable'
+  end
+end
+
+Parallel.each([NotDumpable.new]) do |x|
+  print 'not dumpable'
+  x
+end

data/spec/parallel_spec.rb

@@ -106,6 +106,10 @@ describe Parallel do
     it "returns original array, works like map" do
       `ruby spec/cases/parallel_each.rb`.should == '-b--c--d--a-a b c d'
     end
+
+    it "does not use marshal_dump" do
+      `ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'not dumpable'
+    end
   end
 
   describe :in_groups_of do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: parallel
 version: !ruby/object:Gem::Version 
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors: 
 - Michael Grosser
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-19 00:00:00 +01:00
+date: 2010-01-17 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -27,6 +27,7 @@ files:
 - VERSION
 - lib/parallel.rb
 - parallel.gemspec
+- spec/cases/no_dump_with_each.rb
 - spec/cases/parallel_each.rb
 - spec/cases/parallel_high_fork_rate.rb
 - spec/cases/parallel_influence_outside_data.rb
@@ -76,6 +77,7 @@ test_files:
 - spec/cases/parallel_map_uneven.rb
 - spec/cases/parallel_map_range.rb
 - spec/cases/parallel_with_set_processes.rb
+- spec/cases/no_dump_with_each.rb
 - spec/cases/parallel_start_and_kill.rb
 - spec/cases/parallel_raise.rb
 - spec/cases/parallel_sleeping_2.rb