grosser-parallel 0.3.0 → 0.3.1

data/README.markdown

@@ -1,6 +1,6 @@
 Run any kind of code in parallel Processes or Threads, to speedup computation by factor #{your_cpus} X.
 
- - child processes are killed when your main process is killed through Ctrl+c or kill -2
+ - Child processes are killed when your main process is killed through Ctrl+c or kill -2
 
 Install
 =======
@@ -9,7 +9,7 @@ Install
 Usage
 =====
 ### Processes
- - Speedup through multiple cpus
+ - Speedup through multiple CPUs
  - Speedup for blocking operations
  - Protects global data
  - Extra memory used
@@ -20,7 +20,7 @@ Usage
  - No extra memory used
 
 Map-Reduce-Style
-    # 2 Cpus -> finished after 2 runs (a,b + c)
+    # 2 CPUs -> finished after 2 runs (a,b + c)
     results = Parallel.map(['a','b','c']) do |one_letter|
       expensive_calculation(letter)
     end
@@ -50,10 +50,14 @@ Normal
 
 TODO
 ====
- - optimize Parallel.map by not waiting for a group to finish: start new when one process finishes
+ - JRuby / Windows support <-> possible ?
+
+Authors
+=======
+
+###Contributors (alphabetical)
+ - [TJ Holowaychuk](http://vision-media.ca/) -- tj<$at$>vision-media.ca
 
-Author
-======
 [Michael Grosser](http://pragmatig.wordpress.com)  
 grosser.michael@gmail.com  
 Hereby placed under public domain, do what you want, just do not hold me accountable...

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/lib/parallel.rb

@@ -1,5 +1,7 @@
 class Parallel
-  def self.in_threads(count=2)
+  VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
+
+  def self.in_threads(count = 2)
     out = []
     threads = []
 
@@ -13,29 +15,29 @@ class Parallel
     out
   end
 
-  def self.in_processes(count=nil)
-    count ||= processor_count
-
-    #start writing results into n pipes
+  def self.in_processes(count = processor_count)
+    # Start writing results into n pipes
     reads = []
     writes = []
     pids = []
     count.times do |i|
       reads[i], writes[i] = IO.pipe
-      pids << Process.fork{ Marshal.dump(yield(i), writes[i]) } #write serialized result
+      pids << Process.fork do
+        Marshal.dump(yield(i), writes[i]) # Serialize result
+      end
     end
 
     kill_on_ctrl_c(pids)
 
-    #collect results from pipes simultanously
-    #otherwise pipes get stuck when to much is written (buffer full)
+    # Collect results from pipes simultanously
+    # otherwise pipes get stuck when to much is written (buffer full)
     out = []
     collectors = []
     count.times do |i|
       collectors << Thread.new do
         writes[i].close
 
-        out[i]=""
+        out[i] = ''
         while text = reads[i].gets
           out[i] += text
         end
@@ -44,26 +46,35 @@ class Parallel
       end
     end
 
-    collectors.each{|c|c.join}
+    collectors.each{|c| c.join }
 
-    out.map{|x| Marshal.load(x)} #deserialize
+    out.map{|x| Marshal.load(x) } # Deserialize results
   end
 
-  def self.map(array, options={})
-    count = if options[:in_threads]
-      method = 'in_threads'
-      options[:in_threads]
+  def self.map(array, options = {})
+    require 'thread' # to get Thread.exclusive
+
+    if options[:in_threads]
+      method = :in_threads
+      size = options[method]
     else
-      method = 'in_processes'
-      options[:in_processes] || processor_count
+      method = :in_processes
+      size = options[method] || processor_count
     end
 
+    # work in #{size} threads that use threads/processes
     results = []
-    in_groups_of(array, count).each do |group|
-      results += send(method, group.size) do |i|
-        yield group[i]
+    current = -1
+
+    in_threads(size) do
+      # as long as there are more items, work on one of them
+      loop do
+        index = Thread.exclusive{ current+=1 }
+        break if index >= array.size
+        results[index] = *send(method, 1){ yield array[index] }
       end
     end
+
     results
   end
 
@@ -78,10 +89,10 @@ class Parallel
 
   private
 
-  def self.in_groups_of(array, count)
+  def self.in_groups_of(array, size)
     results = []
     loop do
-      slice = array[(results.size * count)...((results.size+1) * count)]
+      slice = array[(results.size * size)...((results.size+1) * size)]
       if slice.nil? or slice.empty?
         break
       else
@@ -93,10 +104,10 @@ class Parallel
 
   #handle user interrup (Ctrl+c)
   def self.kill_on_ctrl_c(pids)
-    Signal.trap 'SIGINT' do
-      STDERR.puts "Parallel execution interrupted, exiting ..."
-      pids.each { |pid| Process.kill("KILL", pid) }
-      exit 1
+    Signal.trap :SIGINT do
+      $stderr.puts 'Parallel execution interrupted, exiting ...'
+      pids.each { |pid| Process.kill(:KILL, pid) }
+      exit 1 # Quit with 'failed' signal
     end
   end
 end

data/parallel.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{parallel}
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Michael Grosser"]
-  s.date = %q{2009-08-26}
+  s.date = %q{2009-09-26}
   s.email = %q{grosser.michael@gmail.com}
   s.extra_rdoc_files = [
     "README.markdown"
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
      "spec/cases/parallel_influence_outside_data.rb",
      "spec/cases/parallel_map.rb",
      "spec/cases/parallel_map_sleeping.rb",
+     "spec/cases/parallel_map_uneven.rb",
+     "spec/cases/parallel_raise.rb",
      "spec/cases/parallel_sleeping_2.rb",
      "spec/cases/parallel_start_and_kill.rb",
      "spec/cases/parallel_with_detected_cpus.rb",
@@ -35,11 +37,13 @@ Gem::Specification.new do |s|
   s.test_files = [
     "spec/parallel_spec.rb",
      "spec/spec_helper.rb",
+     "spec/cases/parallel_raise.rb",
      "spec/cases/parallel_sleeping_2.rb",
      "spec/cases/parallel_start_and_kill.rb",
      "spec/cases/parallel_with_set_processes.rb",
      "spec/cases/parallel_influence_outside_data.rb",
      "spec/cases/parallel_map_sleeping.rb",
+     "spec/cases/parallel_map_uneven.rb",
      "spec/cases/parallel_with_detected_cpus.rb",
      "spec/cases/parallel_map.rb"
   ]

data/spec/cases/parallel_map_uneven.rb

@@ -0,0 +1,5 @@
+require 'spec/spec_helper.rb'
+
+Parallel.map([1,2,1,2]) do |x|
+  sleep 2 if x == 1
+end

data/spec/cases/parallel_raise.rb

@@ -0,0 +1,10 @@
+require 'spec/spec_helper.rb'
+
+begin
+  Parallel.in_processes(2) do
+    raise "TEST"
+  end
+  puts "FAIL"
+rescue RuntimeError
+  puts $!.message
+end

data/spec/parallel_spec.rb

@@ -37,6 +37,11 @@ describe Parallel do
       `ruby spec/cases/parallel_sleeping_2.rb`
       Time.now.should be_close(t, 3)
     end
+
+    it "raises when one of the processes raises" do
+      pending 'there is some kind of error, but not the original...'
+      `ruby spec/cases/parallel_raise.rb`.should == 'TEST'
+    end
   end
 
   describe :in_threads do
@@ -53,6 +58,10 @@ describe Parallel do
     it "returns results as array" do
       Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
     end
+
+    it "raises when a thread raises" do
+      lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
+    end
   end
 
   describe :map do
@@ -66,6 +75,12 @@ describe Parallel do
       `ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
     end
 
+    it "starts new process imediatly when old exists" do
+      t = Time.now
+      `ruby spec/cases/parallel_map_uneven.rb`
+      Time.now.should be_close(t, 3)
+    end
+
     it "does not flatten results" do
       Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: grosser-parallel
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors: 
 - Michael Grosser
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-26 00:00:00 -07:00
+date: 2009-09-26 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -30,6 +30,8 @@ files:
 - spec/cases/parallel_influence_outside_data.rb
 - spec/cases/parallel_map.rb
 - spec/cases/parallel_map_sleeping.rb
+- spec/cases/parallel_map_uneven.rb
+- spec/cases/parallel_raise.rb
 - spec/cases/parallel_sleeping_2.rb
 - spec/cases/parallel_start_and_kill.rb
 - spec/cases/parallel_with_detected_cpus.rb
@@ -38,6 +40,7 @@ files:
 - spec/spec_helper.rb
 has_rdoc: false
 homepage: http://github.com/grosser/parallel
+licenses: 
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -58,17 +61,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Run any kind of code in parallel processes
 test_files: 
 - spec/parallel_spec.rb
 - spec/spec_helper.rb
+- spec/cases/parallel_raise.rb
 - spec/cases/parallel_sleeping_2.rb
 - spec/cases/parallel_start_and_kill.rb
 - spec/cases/parallel_with_set_processes.rb
 - spec/cases/parallel_influence_outside_data.rb
 - spec/cases/parallel_map_sleeping.rb
+- spec/cases/parallel_map_uneven.rb
 - spec/cases/parallel_with_detected_cpus.rb
 - spec/cases/parallel_map.rb