parallel 0.3.1

data/README.markdown

@@ -0,0 +1,63 @@
+Run any kind of code in parallel Processes or Threads, to speedup computation by factor #{your_cpus} X.
+
+ - Child processes are killed when your main process is killed through Ctrl+c or kill -2
+
+Install
+=======
+    sudo gem install grosser-parallel -s http://gems.github.com/
+
+Usage
+=====
+### Processes
+ - Speedup through multiple CPUs
+ - Speedup for blocking operations
+ - Protects global data
+ - Extra memory used
+
+### Threads
+ - Speedup for blocking operations
+ - Global data can be modified
+ - No extra memory used
+
+Map-Reduce-Style
+    # 2 CPUs -> finished after 2 runs (a,b + c)
+    results = Parallel.map(['a','b','c']) do |one_letter|
+      expensive_calculation(letter)
+    end
+
+    # 3 Processes -> finished after 1 run
+    results = Parallel.map(['a','b','c'], :in_processes=>3){|one_letter| ... }
+
+    # 3 Threads -> finished after 1 run
+    results = Parallel.map(['a','b','c'], :in_threads=>3){|one_letter| ... }
+
+
+Normal
+    #i -> 0...number_of_your_cpus
+    results = Parallel.in_processes do |i|
+      expensive_computation(data[i])
+    end
+
+    #i -> 0...4
+    results = Parallel.in_processes(4) do |i|
+      expensive_computation(data[i])
+    end
+
+    # Threads
+    results = Parallel.in_threads(4) do |i|
+      blocking_computation(data[i])
+    end
+
+TODO
+====
+ - JRuby / Windows support <-> possible ?
+
+Authors
+=======
+
+###Contributors (alphabetical)
+ - [TJ Holowaychuk](http://vision-media.ca/) -- tj<$at$>vision-media.ca
+
+[Michael Grosser](http://pragmatig.wordpress.com)  
+grosser.michael@gmail.com  
+Hereby placed under public domain, do what you want, just do not hold me accountable...

data/Rakefile

@@ -0,0 +1,26 @@
+desc "Run all specs in spec directory"
+task :default do
+  options = "--colour --format progress --loadby --reverse"
+  files = FileList['spec/**/*_spec.rb']
+  system("spec #{options} #{files}")
+end
+
+# fake task so that rubyforge:release works
+task :rdoc do
+end
+
+begin
+  require 'jeweler'
+  project_name = 'parallel'
+  Jeweler::Tasks.new do |gem|
+    gem.name = project_name
+    gem.summary = "Run any kind of code in parallel processes"
+    gem.email = "grosser.michael@gmail.com"
+    gem.homepage = "http://github.com/grosser/#{project_name}"
+    gem.authors = ["Michael Grosser"]
+    gem.rubyforge_project = 'parallel'
+  end
+  Jeweler::RubyforgeTasks.new
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+0.3.1

data/lib/parallel.rb

@@ -0,0 +1,113 @@
+class Parallel
+  VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
+
+  def self.in_threads(count = 2)
+    out = []
+    threads = []
+
+    count.times do |i|
+      threads[i] = Thread.new do
+        out[i] = yield(i)
+      end
+    end
+
+    threads.each{|t| t.join }
+    out
+  end
+
+  def self.in_processes(count = processor_count)
+    # Start writing results into n pipes
+    reads = []
+    writes = []
+    pids = []
+    count.times do |i|
+      reads[i], writes[i] = IO.pipe
+      pids << Process.fork do
+        Marshal.dump(yield(i), writes[i]) # Serialize result
+      end
+    end
+
+    kill_on_ctrl_c(pids)
+
+    # Collect results from pipes simultanously
+    # otherwise pipes get stuck when to much is written (buffer full)
+    out = []
+    collectors = []
+    count.times do |i|
+      collectors << Thread.new do
+        writes[i].close
+
+        out[i] = ''
+        while text = reads[i].gets
+          out[i] += text
+        end
+
+        reads[i].close
+      end
+    end
+
+    collectors.each{|c| c.join }
+
+    out.map{|x| Marshal.load(x) } # Deserialize results
+  end
+
+  def self.map(array, options = {})
+    require 'thread' # to get Thread.exclusive
+
+    if options[:in_threads]
+      method = :in_threads
+      size = options[method]
+    else
+      method = :in_processes
+      size = options[method] || processor_count
+    end
+
+    # work in #{size} threads that use threads/processes
+    results = []
+    current = -1
+
+    in_threads(size) do
+      # as long as there are more items, work on one of them
+      loop do
+        index = Thread.exclusive{ current+=1 }
+        break if index >= array.size
+        results[index] = *send(method, 1){ yield array[index] }
+      end
+    end
+
+    results
+  end
+
+  def self.processor_count
+    case RUBY_PLATFORM
+    when /darwin/
+      `hwprefs cpu_count`.to_i
+    when /linux/
+      `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+    end
+  end
+
+  private
+
+  def self.in_groups_of(array, size)
+    results = []
+    loop do
+      slice = array[(results.size * size)...((results.size+1) * size)]
+      if slice.nil? or slice.empty?
+        break
+      else
+        results << slice
+      end
+    end
+    results
+  end
+
+  #handle user interrup (Ctrl+c)
+  def self.kill_on_ctrl_c(pids)
+    Signal.trap :SIGINT do
+      $stderr.puts 'Parallel execution interrupted, exiting ...'
+      pids.each { |pid| Process.kill(:KILL, pid) }
+      exit 1 # Quit with 'failed' signal
+    end
+  end
+end

data/parallel.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{parallel}
+  s.version = "0.3.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Michael Grosser"]
+  s.date = %q{2009-10-04}
+  s.email = %q{grosser.michael@gmail.com}
+  s.extra_rdoc_files = [
+    "README.markdown"
+  ]
+  s.files = [
+    "README.markdown",
+     "Rakefile",
+     "VERSION",
+     "lib/parallel.rb",
+     "parallel.gemspec",
+     "spec/cases/parallel_influence_outside_data.rb",
+     "spec/cases/parallel_map.rb",
+     "spec/cases/parallel_map_sleeping.rb",
+     "spec/cases/parallel_map_uneven.rb",
+     "spec/cases/parallel_raise.rb",
+     "spec/cases/parallel_sleeping_2.rb",
+     "spec/cases/parallel_start_and_kill.rb",
+     "spec/cases/parallel_with_detected_cpus.rb",
+     "spec/cases/parallel_with_set_processes.rb",
+     "spec/parallel_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/grosser/parallel}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{parallel}
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Run any kind of code in parallel processes}
+  s.test_files = [
+    "spec/parallel_spec.rb",
+     "spec/spec_helper.rb",
+     "spec/cases/parallel_raise.rb",
+     "spec/cases/parallel_sleeping_2.rb",
+     "spec/cases/parallel_start_and_kill.rb",
+     "spec/cases/parallel_with_set_processes.rb",
+     "spec/cases/parallel_influence_outside_data.rb",
+     "spec/cases/parallel_map_sleeping.rb",
+     "spec/cases/parallel_map_uneven.rb",
+     "spec/cases/parallel_with_detected_cpus.rb",
+     "spec/cases/parallel_map.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/spec/cases/parallel_influence_outside_data.rb

@@ -0,0 +1,8 @@
+require 'spec/spec_helper.rb'
+
+x = 'yes'
+
+Parallel.in_processes(2) do
+  x = 'no'
+end
+print x

data/spec/cases/parallel_map.rb

@@ -0,0 +1,6 @@
+require 'spec/spec_helper.rb'
+
+result = Parallel.map(['a','b','c','d']) do |x|
+  "-#{x}-"
+end
+print result * ' '

data/spec/cases/parallel_map_sleeping.rb

@@ -0,0 +1,5 @@
+require 'spec/spec_helper.rb'
+
+Parallel.map(['a','b','c','d']) do |x|
+  sleep 1
+end

data/spec/cases/parallel_map_uneven.rb

@@ -0,0 +1,5 @@
+require 'spec/spec_helper.rb'
+
+Parallel.map([1,2,1,2]) do |x|
+  sleep 2 if x == 1
+end

data/spec/cases/parallel_raise.rb

@@ -0,0 +1,10 @@
+require 'spec/spec_helper.rb'
+
+begin
+  Parallel.in_processes(2) do
+    raise "TEST"
+  end
+  puts "FAIL"
+rescue RuntimeError
+  puts $!.message
+end

data/spec/cases/parallel_sleeping_2.rb

@@ -0,0 +1,5 @@
+require 'spec/spec_helper.rb'
+
+Parallel.in_processes(5) do
+  sleep 2
+end

data/spec/cases/parallel_start_and_kill.rb

@@ -0,0 +1,6 @@
+require 'spec/spec_helper.rb'
+
+Parallel.in_processes(2) do
+  sleep 10
+  puts "I should have been killed earlier..."
+end

data/spec/cases/parallel_with_detected_cpus.rb

@@ -0,0 +1,6 @@
+require 'spec/spec_helper.rb'
+
+x = Parallel.in_processes do
+  "HELLO"
+end
+puts x

data/spec/cases/parallel_with_set_processes.rb

@@ -0,0 +1,6 @@
+require 'spec/spec_helper.rb'
+
+x = Parallel.in_processes(5) do
+  "HELLO"
+end
+puts x

data/spec/parallel_spec.rb

@@ -0,0 +1,110 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Parallel do
+  describe :in_processes do
+    before do
+      @cpus = Parallel.processor_count
+    end
+
+    it "executes with detected cpus" do
+      `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * @cpus
+    end
+
+    it "set ammount of parallel processes" do
+      `ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
+    end
+
+    it "does not influence outside data" do
+      `ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
+    end
+
+    it "kills the processes when the main process gets killed through ctrl+c" do
+      t = Time.now
+      lambda{
+        Thread.new do
+          `ruby spec/cases/parallel_start_and_kill.rb`
+        end
+        sleep 1
+        running_processes = `ps -f`.split("\n").map{|line| line.split(/\s+/)}
+        parent = running_processes.detect{|line| line.include?("00:00:00") and line.include?("ruby") }[1]
+        `kill -2 #{parent}` #simulates Ctrl+c
+      }.should_not change{`ps`.split("\n").size}
+      Time.now.should be_close(t, 3)
+    end
+
+    it "saves time" do
+      t = Time.now
+      `ruby spec/cases/parallel_sleeping_2.rb`
+      Time.now.should be_close(t, 3)
+    end
+
+    it "raises when one of the processes raises" do
+      pending 'there is some kind of error, but not the original...'
+      `ruby spec/cases/parallel_raise.rb`.should == 'TEST'
+    end
+  end
+
+  describe :in_threads do
+    it "saves time" do
+      t = Time.now
+      Parallel.in_threads(3){ sleep 2 }
+      Time.now.should be_close(t, 3)
+    end
+
+    it "does not create new processes" do
+      lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
+    end
+
+    it "returns results as array" do
+      Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
+    end
+
+    it "raises when a thread raises" do
+      lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
+    end
+  end
+
+  describe :map do
+    it "saves time" do
+      t = Time.now
+      `ruby spec/cases/parallel_map_sleeping.rb`
+      Time.now.should be_close(t, 3)
+    end
+
+    it "executes with given parameters" do
+      `ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
+    end
+
+    it "starts new process imediatly when old exists" do
+      t = Time.now
+      `ruby spec/cases/parallel_map_uneven.rb`
+      Time.now.should be_close(t, 3)
+    end
+
+    it "does not flatten results" do
+      Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
+    end
+
+    it "can run in threads" do
+      Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+    end
+  end
+
+  describe :in_groups_of do
+    it "works for empty" do
+      Parallel.send(:in_groups_of, [], 3).should == []
+    end
+
+    it "works for smaller then count" do
+      Parallel.send(:in_groups_of, [1,2], 3).should == [[1,2]]
+    end
+
+    it "works for count" do
+      Parallel.send(:in_groups_of, [1,2,3], 3).should == [[1,2,3]]
+    end
+
+    it "works for larger than count" do
+      Parallel.send(:in_groups_of, [1,2,3,4], 3).should == [[1,2,3],[4]]
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+# ---- requirements
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+
+require 'parallel'

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: parallel
+version: !ruby/object:Gem::Version 
+  version: 0.3.1
+platform: ruby
+authors: 
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-04 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: 
+email: grosser.michael@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- README.markdown
+- Rakefile
+- VERSION
+- lib/parallel.rb
+- parallel.gemspec
+- spec/cases/parallel_influence_outside_data.rb
+- spec/cases/parallel_map.rb
+- spec/cases/parallel_map_sleeping.rb
+- spec/cases/parallel_map_uneven.rb
+- spec/cases/parallel_raise.rb
+- spec/cases/parallel_sleeping_2.rb
+- spec/cases/parallel_start_and_kill.rb
+- spec/cases/parallel_with_detected_cpus.rb
+- spec/cases/parallel_with_set_processes.rb
+- spec/parallel_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/grosser/parallel
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: parallel
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Run any kind of code in parallel processes
+test_files: 
+- spec/parallel_spec.rb
+- spec/spec_helper.rb
+- spec/cases/parallel_raise.rb
+- spec/cases/parallel_sleeping_2.rb
+- spec/cases/parallel_start_and_kill.rb
+- spec/cases/parallel_with_set_processes.rb
+- spec/cases/parallel_influence_outside_data.rb
+- spec/cases/parallel_map_sleeping.rb
+- spec/cases/parallel_map_uneven.rb
+- spec/cases/parallel_with_detected_cpus.rb
+- spec/cases/parallel_map.rb