grosser-parallel 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
data/README.markdown ADDED
@@ -0,0 +1,31 @@
1
+ Run any kind of code in parallel Processes or Threads, to speedup computation by factor #{your_cpus} X.
2
+
3
+ - child processes are killed when your main process is killed through Ctrl+c or kill -2
4
+
5
+ Install
6
+ =======
7
+ sudo gem install grosser-parallel -s http://gems.github.com/
8
+
9
+ Usage
10
+ =====
11
+
12
+ #i -> 0..number_of_your_cpus
13
+ results = Parallel.in_processes do |i|
14
+ expensive_computation(data[i])
15
+ end
16
+
17
+ #i -> 0..4
18
+ results = Parallel.in_processes(4) do |i|
19
+ expensive_computation(data[i])
20
+ end
21
+
22
+ #same with threads (no speedup through multiple cpus, but speedup for blocking operations)
23
+ results = Parallel.in_threads(4) do |i|
24
+ blocking_computation(data[i])
25
+ end
26
+
27
+ Author
28
+ ======
29
+ [Michael Grosser](http://pragmatig.wordpress.com)
30
+ grosser.michael@gmail.com
31
+ Hereby placed under public domain, do what you want, just do not hold me accountable...
data/Rakefile ADDED
@@ -0,0 +1,21 @@
1
+ desc "Run all specs in spec directory"
2
+ task :default do
3
+ options = "--colour --format progress --loadby --reverse"
4
+ files = FileList['spec/**/*_spec.rb']
5
+ system("spec #{options} #{files}")
6
+ end
7
+
8
+
9
+ begin
10
+ require 'jeweler'
11
+ project_name = 'parallel'
12
+ Jeweler::Tasks.new do |gem|
13
+ gem.name = project_name
14
+ gem.summary = "Run any kind of code in parallel processes"
15
+ gem.email = "grosser.michael@gmail.com"
16
+ gem.homepage = "http://github.com/grosser/#{project_name}"
17
+ gem.authors = ["Michael Grosser"]
18
+ end
19
+ rescue LoadError
20
+ puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
21
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.0
data/lib/parallel.rb ADDED
@@ -0,0 +1,71 @@
1
+ class Parallel
2
+ def self.in_threads(count=2)
3
+ out = []
4
+ threads = []
5
+
6
+ count.times do |i|
7
+ threads[i] = Thread.new do
8
+ out[i] = yield(i)
9
+ end
10
+ end
11
+
12
+ threads.each{|t| t.join }
13
+ out
14
+ end
15
+
16
+ def self.in_processes(count=nil)
17
+ count ||= processor_count
18
+
19
+ #start writing results into n pipes
20
+ reads = []
21
+ writes = []
22
+ pids = []
23
+ count.times do |i|
24
+ reads[i], writes[i] = IO.pipe
25
+ pids << Process.fork{ Marshal.dump(yield(i), writes[i]) } #write serialized result
26
+ end
27
+
28
+ kill_on_ctrl_c(pids)
29
+
30
+ #collect results from pipes simultanously
31
+ #otherwise pipes get stuck when to much is written (buffer full)
32
+ out = []
33
+ collectors = []
34
+ count.times do |i|
35
+ collectors << Thread.new do
36
+ writes[i].close
37
+
38
+ out[i]=""
39
+ while text = reads[i].gets
40
+ out[i] += text
41
+ end
42
+
43
+ reads[i].close
44
+ end
45
+ end
46
+
47
+ collectors.each{|c|c.join}
48
+
49
+ out.map{|x| Marshal.load(x)} #deserialize
50
+ end
51
+
52
+ def self.processor_count
53
+ case RUBY_PLATFORM
54
+ when /darwin/
55
+ `hwprefs cpu_count`.to_i
56
+ when /linux/
57
+ `cat /proc/cpuinfo | grep processor | wc -l`.to_i
58
+ end
59
+ end
60
+
61
+ private
62
+
63
+ #handle user interrup (Ctrl+c)
64
+ def self.kill_on_ctrl_c(pids)
65
+ Signal.trap 'SIGINT' do
66
+ STDERR.puts "Parallel execution interrupted, exiting ..."
67
+ pids.each { |pid| Process.kill("KILL", pid) }
68
+ exit 1
69
+ end
70
+ end
71
+ end
data/parallel.gemspec ADDED
@@ -0,0 +1,52 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = %q{parallel}
5
+ s.version = "0.2.0"
6
+
7
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
8
+ s.authors = ["Michael Grosser"]
9
+ s.date = %q{2009-08-13}
10
+ s.email = %q{grosser.michael@gmail.com}
11
+ s.extra_rdoc_files = [
12
+ "README.markdown"
13
+ ]
14
+ s.files = [
15
+ "README.markdown",
16
+ "Rakefile",
17
+ "VERSION",
18
+ "lib/parallel.rb",
19
+ "parallel.gemspec",
20
+ "spec/cases/parallel_influence_outside_data.rb",
21
+ "spec/cases/parallel_sleeping_2.rb",
22
+ "spec/cases/parallel_start_and_kill.rb",
23
+ "spec/cases/parallel_with_detected_cpus.rb",
24
+ "spec/cases/parallel_with_set_processes.rb",
25
+ "spec/parallel_spec.rb",
26
+ "spec/spec_helper.rb"
27
+ ]
28
+ s.homepage = %q{http://github.com/grosser/parallel}
29
+ s.rdoc_options = ["--charset=UTF-8"]
30
+ s.require_paths = ["lib"]
31
+ s.rubygems_version = %q{1.3.4}
32
+ s.summary = %q{Run any kind of code in parallel processes}
33
+ s.test_files = [
34
+ "spec/parallel_spec.rb",
35
+ "spec/spec_helper.rb",
36
+ "spec/cases/parallel_sleeping_2.rb",
37
+ "spec/cases/parallel_start_and_kill.rb",
38
+ "spec/cases/parallel_with_set_processes.rb",
39
+ "spec/cases/parallel_influence_outside_data.rb",
40
+ "spec/cases/parallel_with_detected_cpus.rb"
41
+ ]
42
+
43
+ if s.respond_to? :specification_version then
44
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
45
+ s.specification_version = 3
46
+
47
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
48
+ else
49
+ end
50
+ else
51
+ end
52
+ end
@@ -0,0 +1,8 @@
1
+ require 'spec/spec_helper.rb'
2
+
3
+ x = 'yes'
4
+
5
+ Parallel.in_processes(2) do
6
+ x = 'no'
7
+ end
8
+ print x
@@ -0,0 +1,5 @@
1
+ require 'spec/spec_helper.rb'
2
+
3
+ Parallel.in_processes(5) do
4
+ sleep 2
5
+ end
@@ -0,0 +1,6 @@
1
+ require 'spec/spec_helper.rb'
2
+
3
+ Parallel.in_processes(2) do
4
+ sleep 10
5
+ puts "I should have been killed earlier..."
6
+ end
@@ -0,0 +1,6 @@
1
+ require 'spec/spec_helper.rb'
2
+
3
+ x = Parallel.in_processes do
4
+ "HELLO"
5
+ end
6
+ puts x
@@ -0,0 +1,6 @@
1
+ require 'spec/spec_helper.rb'
2
+
3
+ x = Parallel.in_processes(5) do
4
+ "HELLO"
5
+ end
6
+ puts x
@@ -0,0 +1,57 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe Parallel do
4
+ describe :in_processes do
5
+ before do
6
+ @cpus = Parallel.processor_count
7
+ end
8
+
9
+ it "executes with detected cpus" do
10
+ `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * @cpus
11
+ end
12
+
13
+ it "set ammount of parallel processes" do
14
+ `ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
15
+ end
16
+
17
+ it "does not influence outside data" do
18
+ `ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
19
+ end
20
+
21
+ it "kills the processes when the main process gets killed through ctrl+c" do
22
+ t = Time.now
23
+ lambda{
24
+ Thread.new do
25
+ `ruby spec/cases/parallel_start_and_kill.rb`
26
+ end
27
+ sleep 1
28
+ running_processes = `ps -f`.split("\n").map{|line| line.split(/\s+/)}
29
+ parent = running_processes.detect{|line| line.include?("00:00:00") and line.include?("ruby") }[1]
30
+ `kill -2 #{parent}` #simulates Ctrl+c
31
+ }.should_not change{`ps`.split("\n").size}
32
+ Time.now.should be_close(t, 3)
33
+ end
34
+
35
+ it "saves time" do
36
+ t = Time.now
37
+ `ruby spec/cases/parallel_sleeping_2.rb`
38
+ Time.now.should be_close(t, 3)
39
+ end
40
+ end
41
+
42
+ describe :in_threads do
43
+ it "saves time" do
44
+ t = Time.now
45
+ Parallel.in_threads(3){ sleep 2 }
46
+ Time.now.should be_close(t, 3)
47
+ end
48
+
49
+ it "does not create new processes" do
50
+ lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
51
+ end
52
+
53
+ it "returns results as array" do
54
+ Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,4 @@
1
+ # ---- requirements
2
+ $LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
3
+
4
+ require 'parallel'
metadata ADDED
@@ -0,0 +1,71 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: grosser-parallel
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Michael Grosser
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-08-13 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description:
17
+ email: grosser.michael@gmail.com
18
+ executables: []
19
+
20
+ extensions: []
21
+
22
+ extra_rdoc_files:
23
+ - README.markdown
24
+ files:
25
+ - README.markdown
26
+ - Rakefile
27
+ - VERSION
28
+ - lib/parallel.rb
29
+ - parallel.gemspec
30
+ - spec/cases/parallel_influence_outside_data.rb
31
+ - spec/cases/parallel_sleeping_2.rb
32
+ - spec/cases/parallel_start_and_kill.rb
33
+ - spec/cases/parallel_with_detected_cpus.rb
34
+ - spec/cases/parallel_with_set_processes.rb
35
+ - spec/parallel_spec.rb
36
+ - spec/spec_helper.rb
37
+ has_rdoc: false
38
+ homepage: http://github.com/grosser/parallel
39
+ licenses:
40
+ post_install_message:
41
+ rdoc_options:
42
+ - --charset=UTF-8
43
+ require_paths:
44
+ - lib
45
+ required_ruby_version: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ version: "0"
50
+ version:
51
+ required_rubygems_version: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: "0"
56
+ version:
57
+ requirements: []
58
+
59
+ rubyforge_project:
60
+ rubygems_version: 1.3.5
61
+ signing_key:
62
+ specification_version: 3
63
+ summary: Run any kind of code in parallel processes
64
+ test_files:
65
+ - spec/parallel_spec.rb
66
+ - spec/spec_helper.rb
67
+ - spec/cases/parallel_sleeping_2.rb
68
+ - spec/cases/parallel_start_and_kill.rb
69
+ - spec/cases/parallel_with_set_processes.rb
70
+ - spec/cases/parallel_influence_outside_data.rb
71
+ - spec/cases/parallel_with_detected_cpus.rb