grosser-parallel 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +10 -6
- data/VERSION +1 -1
- data/lib/parallel.rb +37 -26
- data/parallel.gemspec +6 -2
- data/spec/cases/parallel_map_uneven.rb +5 -0
- data/spec/cases/parallel_raise.rb +10 -0
- data/spec/parallel_spec.rb +15 -0
- metadata +8 -3
data/README.markdown
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Run any kind of code in parallel Processes or Threads, to speedup computation by factor #{your_cpus} X.
|
2
2
|
|
3
|
-
-
|
3
|
+
- Child processes are killed when your main process is killed through Ctrl+c or kill -2
|
4
4
|
|
5
5
|
Install
|
6
6
|
=======
|
@@ -9,7 +9,7 @@ Install
|
|
9
9
|
Usage
|
10
10
|
=====
|
11
11
|
### Processes
|
12
|
-
- Speedup through multiple
|
12
|
+
- Speedup through multiple CPUs
|
13
13
|
- Speedup for blocking operations
|
14
14
|
- Protects global data
|
15
15
|
- Extra memory used
|
@@ -20,7 +20,7 @@ Usage
|
|
20
20
|
- No extra memory used
|
21
21
|
|
22
22
|
Map-Reduce-Style
|
23
|
-
# 2
|
23
|
+
# 2 CPUs -> finished after 2 runs (a,b + c)
|
24
24
|
results = Parallel.map(['a','b','c']) do |one_letter|
|
25
25
|
expensive_calculation(letter)
|
26
26
|
end
|
@@ -50,10 +50,14 @@ Normal
|
|
50
50
|
|
51
51
|
TODO
|
52
52
|
====
|
53
|
-
-
|
53
|
+
- JRuby / Windows support <-> possible ?
|
54
|
+
|
55
|
+
Authors
|
56
|
+
=======
|
57
|
+
|
58
|
+
###Contributors (alphabetical)
|
59
|
+
- [TJ Holowaychuk](http://vision-media.ca/) -- tj<$at$>vision-media.ca
|
54
60
|
|
55
|
-
Author
|
56
|
-
======
|
57
61
|
[Michael Grosser](http://pragmatig.wordpress.com)
|
58
62
|
grosser.michael@gmail.com
|
59
63
|
Hereby placed under public domain, do what you want, just do not hold me accountable...
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.1
|
data/lib/parallel.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
class Parallel
|
2
|
-
|
2
|
+
VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
|
3
|
+
|
4
|
+
def self.in_threads(count = 2)
|
3
5
|
out = []
|
4
6
|
threads = []
|
5
7
|
|
@@ -13,29 +15,29 @@ class Parallel
|
|
13
15
|
out
|
14
16
|
end
|
15
17
|
|
16
|
-
def self.in_processes(count=
|
17
|
-
|
18
|
-
|
19
|
-
#start writing results into n pipes
|
18
|
+
def self.in_processes(count = processor_count)
|
19
|
+
# Start writing results into n pipes
|
20
20
|
reads = []
|
21
21
|
writes = []
|
22
22
|
pids = []
|
23
23
|
count.times do |i|
|
24
24
|
reads[i], writes[i] = IO.pipe
|
25
|
-
pids << Process.fork
|
25
|
+
pids << Process.fork do
|
26
|
+
Marshal.dump(yield(i), writes[i]) # Serialize result
|
27
|
+
end
|
26
28
|
end
|
27
29
|
|
28
30
|
kill_on_ctrl_c(pids)
|
29
31
|
|
30
|
-
#
|
31
|
-
#otherwise pipes get stuck when to much is written (buffer full)
|
32
|
+
# Collect results from pipes simultanously
|
33
|
+
# otherwise pipes get stuck when to much is written (buffer full)
|
32
34
|
out = []
|
33
35
|
collectors = []
|
34
36
|
count.times do |i|
|
35
37
|
collectors << Thread.new do
|
36
38
|
writes[i].close
|
37
39
|
|
38
|
-
out[i]=
|
40
|
+
out[i] = ''
|
39
41
|
while text = reads[i].gets
|
40
42
|
out[i] += text
|
41
43
|
end
|
@@ -44,26 +46,35 @@ class Parallel
|
|
44
46
|
end
|
45
47
|
end
|
46
48
|
|
47
|
-
collectors.each{|c|c.join}
|
49
|
+
collectors.each{|c| c.join }
|
48
50
|
|
49
|
-
out.map{|x| Marshal.load(x)} #
|
51
|
+
out.map{|x| Marshal.load(x) } # Deserialize results
|
50
52
|
end
|
51
53
|
|
52
|
-
def self.map(array, options={})
|
53
|
-
|
54
|
-
|
55
|
-
|
54
|
+
def self.map(array, options = {})
|
55
|
+
require 'thread' # to get Thread.exclusive
|
56
|
+
|
57
|
+
if options[:in_threads]
|
58
|
+
method = :in_threads
|
59
|
+
size = options[method]
|
56
60
|
else
|
57
|
-
method =
|
58
|
-
options[
|
61
|
+
method = :in_processes
|
62
|
+
size = options[method] || processor_count
|
59
63
|
end
|
60
64
|
|
65
|
+
# work in #{size} threads that use threads/processes
|
61
66
|
results = []
|
62
|
-
|
63
|
-
|
64
|
-
|
67
|
+
current = -1
|
68
|
+
|
69
|
+
in_threads(size) do
|
70
|
+
# as long as there are more items, work on one of them
|
71
|
+
loop do
|
72
|
+
index = Thread.exclusive{ current+=1 }
|
73
|
+
break if index >= array.size
|
74
|
+
results[index] = *send(method, 1){ yield array[index] }
|
65
75
|
end
|
66
76
|
end
|
77
|
+
|
67
78
|
results
|
68
79
|
end
|
69
80
|
|
@@ -78,10 +89,10 @@ class Parallel
|
|
78
89
|
|
79
90
|
private
|
80
91
|
|
81
|
-
def self.in_groups_of(array,
|
92
|
+
def self.in_groups_of(array, size)
|
82
93
|
results = []
|
83
94
|
loop do
|
84
|
-
slice = array[(results.size *
|
95
|
+
slice = array[(results.size * size)...((results.size+1) * size)]
|
85
96
|
if slice.nil? or slice.empty?
|
86
97
|
break
|
87
98
|
else
|
@@ -93,10 +104,10 @@ class Parallel
|
|
93
104
|
|
94
105
|
#handle user interrup (Ctrl+c)
|
95
106
|
def self.kill_on_ctrl_c(pids)
|
96
|
-
Signal.trap
|
97
|
-
|
98
|
-
pids.each { |pid| Process.kill(
|
99
|
-
exit 1
|
107
|
+
Signal.trap :SIGINT do
|
108
|
+
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
109
|
+
pids.each { |pid| Process.kill(:KILL, pid) }
|
110
|
+
exit 1 # Quit with 'failed' signal
|
100
111
|
end
|
101
112
|
end
|
102
113
|
end
|
data/parallel.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{parallel}
|
5
|
-
s.version = "0.3.
|
5
|
+
s.version = "0.3.1"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Michael Grosser"]
|
9
|
-
s.date = %q{2009-
|
9
|
+
s.date = %q{2009-09-26}
|
10
10
|
s.email = %q{grosser.michael@gmail.com}
|
11
11
|
s.extra_rdoc_files = [
|
12
12
|
"README.markdown"
|
@@ -20,6 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
"spec/cases/parallel_influence_outside_data.rb",
|
21
21
|
"spec/cases/parallel_map.rb",
|
22
22
|
"spec/cases/parallel_map_sleeping.rb",
|
23
|
+
"spec/cases/parallel_map_uneven.rb",
|
24
|
+
"spec/cases/parallel_raise.rb",
|
23
25
|
"spec/cases/parallel_sleeping_2.rb",
|
24
26
|
"spec/cases/parallel_start_and_kill.rb",
|
25
27
|
"spec/cases/parallel_with_detected_cpus.rb",
|
@@ -35,11 +37,13 @@ Gem::Specification.new do |s|
|
|
35
37
|
s.test_files = [
|
36
38
|
"spec/parallel_spec.rb",
|
37
39
|
"spec/spec_helper.rb",
|
40
|
+
"spec/cases/parallel_raise.rb",
|
38
41
|
"spec/cases/parallel_sleeping_2.rb",
|
39
42
|
"spec/cases/parallel_start_and_kill.rb",
|
40
43
|
"spec/cases/parallel_with_set_processes.rb",
|
41
44
|
"spec/cases/parallel_influence_outside_data.rb",
|
42
45
|
"spec/cases/parallel_map_sleeping.rb",
|
46
|
+
"spec/cases/parallel_map_uneven.rb",
|
43
47
|
"spec/cases/parallel_with_detected_cpus.rb",
|
44
48
|
"spec/cases/parallel_map.rb"
|
45
49
|
]
|
data/spec/parallel_spec.rb
CHANGED
@@ -37,6 +37,11 @@ describe Parallel do
|
|
37
37
|
`ruby spec/cases/parallel_sleeping_2.rb`
|
38
38
|
Time.now.should be_close(t, 3)
|
39
39
|
end
|
40
|
+
|
41
|
+
it "raises when one of the processes raises" do
|
42
|
+
pending 'there is some kind of error, but not the original...'
|
43
|
+
`ruby spec/cases/parallel_raise.rb`.should == 'TEST'
|
44
|
+
end
|
40
45
|
end
|
41
46
|
|
42
47
|
describe :in_threads do
|
@@ -53,6 +58,10 @@ describe Parallel do
|
|
53
58
|
it "returns results as array" do
|
54
59
|
Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
|
55
60
|
end
|
61
|
+
|
62
|
+
it "raises when a thread raises" do
|
63
|
+
lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
|
64
|
+
end
|
56
65
|
end
|
57
66
|
|
58
67
|
describe :map do
|
@@ -66,6 +75,12 @@ describe Parallel do
|
|
66
75
|
`ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
|
67
76
|
end
|
68
77
|
|
78
|
+
it "starts new process imediatly when old exists" do
|
79
|
+
t = Time.now
|
80
|
+
`ruby spec/cases/parallel_map_uneven.rb`
|
81
|
+
Time.now.should be_close(t, 3)
|
82
|
+
end
|
83
|
+
|
69
84
|
it "does not flatten results" do
|
70
85
|
Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
|
71
86
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: grosser-parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
12
|
+
date: 2009-09-26 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -30,6 +30,8 @@ files:
|
|
30
30
|
- spec/cases/parallel_influence_outside_data.rb
|
31
31
|
- spec/cases/parallel_map.rb
|
32
32
|
- spec/cases/parallel_map_sleeping.rb
|
33
|
+
- spec/cases/parallel_map_uneven.rb
|
34
|
+
- spec/cases/parallel_raise.rb
|
33
35
|
- spec/cases/parallel_sleeping_2.rb
|
34
36
|
- spec/cases/parallel_start_and_kill.rb
|
35
37
|
- spec/cases/parallel_with_detected_cpus.rb
|
@@ -38,6 +40,7 @@ files:
|
|
38
40
|
- spec/spec_helper.rb
|
39
41
|
has_rdoc: false
|
40
42
|
homepage: http://github.com/grosser/parallel
|
43
|
+
licenses:
|
41
44
|
post_install_message:
|
42
45
|
rdoc_options:
|
43
46
|
- --charset=UTF-8
|
@@ -58,17 +61,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
58
61
|
requirements: []
|
59
62
|
|
60
63
|
rubyforge_project:
|
61
|
-
rubygems_version: 1.
|
64
|
+
rubygems_version: 1.3.5
|
62
65
|
signing_key:
|
63
66
|
specification_version: 3
|
64
67
|
summary: Run any kind of code in parallel processes
|
65
68
|
test_files:
|
66
69
|
- spec/parallel_spec.rb
|
67
70
|
- spec/spec_helper.rb
|
71
|
+
- spec/cases/parallel_raise.rb
|
68
72
|
- spec/cases/parallel_sleeping_2.rb
|
69
73
|
- spec/cases/parallel_start_and_kill.rb
|
70
74
|
- spec/cases/parallel_with_set_processes.rb
|
71
75
|
- spec/cases/parallel_influence_outside_data.rb
|
72
76
|
- spec/cases/parallel_map_sleeping.rb
|
77
|
+
- spec/cases/parallel_map_uneven.rb
|
73
78
|
- spec/cases/parallel_with_detected_cpus.rb
|
74
79
|
- spec/cases/parallel_map.rb
|