grosser-parallel 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +31 -3
- data/VERSION +1 -1
- data/lib/parallel.rb +31 -0
- data/parallel.gemspec +7 -3
- data/spec/cases/parallel_map.rb +6 -0
- data/spec/cases/parallel_map_sleeping.rb +5 -0
- data/spec/parallel_spec.rb +38 -0
- metadata +7 -4
data/README.markdown
CHANGED
@@ -8,22 +8,50 @@ Install
|
|
8
8
|
|
9
9
|
Usage
|
10
10
|
=====
|
11
|
+
### Processes
|
12
|
+
- Speedup through multiple cpus
|
13
|
+
- Speedup for blocking operations
|
14
|
+
- Protects global data
|
15
|
+
- Extra memory used
|
11
16
|
|
12
|
-
|
17
|
+
### Threads
|
18
|
+
- Speedup for blocking operations
|
19
|
+
- Global data can be modified
|
20
|
+
- No extra memory used
|
21
|
+
|
22
|
+
Map-Reduce-Style
|
23
|
+
# 2 Cpus -> finished after 2 runs (a,b + c)
|
24
|
+
results = Parallel.map(['a','b','c']) do |one_letter|
|
25
|
+
expensive_calculation(letter)
|
26
|
+
end
|
27
|
+
|
28
|
+
# 3 Processes -> finished after 1 run
|
29
|
+
results = Parallel.map(['a','b','c'], :in_processes=>3){|one_letter| ... }
|
30
|
+
|
31
|
+
# 3 Threads -> finished after 1 run
|
32
|
+
results = Parallel.map(['a','b','c'], :in_threads=>3){|one_letter| ... }
|
33
|
+
|
34
|
+
|
35
|
+
Normal
|
36
|
+
#i -> 0...number_of_your_cpus
|
13
37
|
results = Parallel.in_processes do |i|
|
14
38
|
expensive_computation(data[i])
|
15
39
|
end
|
16
40
|
|
17
|
-
#i -> 0
|
41
|
+
#i -> 0...4
|
18
42
|
results = Parallel.in_processes(4) do |i|
|
19
43
|
expensive_computation(data[i])
|
20
44
|
end
|
21
45
|
|
22
|
-
#
|
46
|
+
# Threads
|
23
47
|
results = Parallel.in_threads(4) do |i|
|
24
48
|
blocking_computation(data[i])
|
25
49
|
end
|
26
50
|
|
51
|
+
TODO
|
52
|
+
====
|
53
|
+
- optimize Parallel.map by not waiting for a group to finish: start new when one process finishes
|
54
|
+
|
27
55
|
Author
|
28
56
|
======
|
29
57
|
[Michael Grosser](http://pragmatig.wordpress.com)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
data/lib/parallel.rb
CHANGED
@@ -49,6 +49,24 @@ class Parallel
|
|
49
49
|
out.map{|x| Marshal.load(x)} #deserialize
|
50
50
|
end
|
51
51
|
|
52
|
+
def self.map(array, options={})
|
53
|
+
count = if options[:in_threads]
|
54
|
+
method = 'in_threads'
|
55
|
+
options[:in_threads]
|
56
|
+
else
|
57
|
+
method = 'in_processes'
|
58
|
+
options[:in_processes] || processor_count
|
59
|
+
end
|
60
|
+
|
61
|
+
results = []
|
62
|
+
in_groups_of(array, count).each do |group|
|
63
|
+
results += send(method, group.size) do |i|
|
64
|
+
yield group[i]
|
65
|
+
end
|
66
|
+
end
|
67
|
+
results
|
68
|
+
end
|
69
|
+
|
52
70
|
def self.processor_count
|
53
71
|
case RUBY_PLATFORM
|
54
72
|
when /darwin/
|
@@ -60,6 +78,19 @@ class Parallel
|
|
60
78
|
|
61
79
|
private
|
62
80
|
|
81
|
+
def self.in_groups_of(array, count)
|
82
|
+
results = []
|
83
|
+
loop do
|
84
|
+
slice = array[(results.size * count)...((results.size+1) * count)]
|
85
|
+
if slice.nil? or slice.empty?
|
86
|
+
break
|
87
|
+
else
|
88
|
+
results << slice
|
89
|
+
end
|
90
|
+
end
|
91
|
+
results
|
92
|
+
end
|
93
|
+
|
63
94
|
#handle user interrup (Ctrl+c)
|
64
95
|
def self.kill_on_ctrl_c(pids)
|
65
96
|
Signal.trap 'SIGINT' do
|
data/parallel.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{parallel}
|
5
|
-
s.version = "0.
|
5
|
+
s.version = "0.3.0"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Michael Grosser"]
|
9
|
-
s.date = %q{2009-08-
|
9
|
+
s.date = %q{2009-08-26}
|
10
10
|
s.email = %q{grosser.michael@gmail.com}
|
11
11
|
s.extra_rdoc_files = [
|
12
12
|
"README.markdown"
|
@@ -18,6 +18,8 @@ Gem::Specification.new do |s|
|
|
18
18
|
"lib/parallel.rb",
|
19
19
|
"parallel.gemspec",
|
20
20
|
"spec/cases/parallel_influence_outside_data.rb",
|
21
|
+
"spec/cases/parallel_map.rb",
|
22
|
+
"spec/cases/parallel_map_sleeping.rb",
|
21
23
|
"spec/cases/parallel_sleeping_2.rb",
|
22
24
|
"spec/cases/parallel_start_and_kill.rb",
|
23
25
|
"spec/cases/parallel_with_detected_cpus.rb",
|
@@ -37,7 +39,9 @@ Gem::Specification.new do |s|
|
|
37
39
|
"spec/cases/parallel_start_and_kill.rb",
|
38
40
|
"spec/cases/parallel_with_set_processes.rb",
|
39
41
|
"spec/cases/parallel_influence_outside_data.rb",
|
40
|
-
"spec/cases/
|
42
|
+
"spec/cases/parallel_map_sleeping.rb",
|
43
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
44
|
+
"spec/cases/parallel_map.rb"
|
41
45
|
]
|
42
46
|
|
43
47
|
if s.respond_to? :specification_version then
|
data/spec/parallel_spec.rb
CHANGED
@@ -54,4 +54,42 @@ describe Parallel do
|
|
54
54
|
Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
|
55
55
|
end
|
56
56
|
end
|
57
|
+
|
58
|
+
describe :map do
|
59
|
+
it "saves time" do
|
60
|
+
t = Time.now
|
61
|
+
`ruby spec/cases/parallel_map_sleeping.rb`
|
62
|
+
Time.now.should be_close(t, 3)
|
63
|
+
end
|
64
|
+
|
65
|
+
it "executes with given parameters" do
|
66
|
+
`ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
|
67
|
+
end
|
68
|
+
|
69
|
+
it "does not flatten results" do
|
70
|
+
Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
|
71
|
+
end
|
72
|
+
|
73
|
+
it "can run in threads" do
|
74
|
+
Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
describe :in_groups_of do
|
79
|
+
it "works for empty" do
|
80
|
+
Parallel.send(:in_groups_of, [], 3).should == []
|
81
|
+
end
|
82
|
+
|
83
|
+
it "works for smaller then count" do
|
84
|
+
Parallel.send(:in_groups_of, [1,2], 3).should == [[1,2]]
|
85
|
+
end
|
86
|
+
|
87
|
+
it "works for count" do
|
88
|
+
Parallel.send(:in_groups_of, [1,2,3], 3).should == [[1,2,3]]
|
89
|
+
end
|
90
|
+
|
91
|
+
it "works for larger than count" do
|
92
|
+
Parallel.send(:in_groups_of, [1,2,3,4], 3).should == [[1,2,3],[4]]
|
93
|
+
end
|
94
|
+
end
|
57
95
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: grosser-parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-08-
|
12
|
+
date: 2009-08-26 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -28,6 +28,8 @@ files:
|
|
28
28
|
- lib/parallel.rb
|
29
29
|
- parallel.gemspec
|
30
30
|
- spec/cases/parallel_influence_outside_data.rb
|
31
|
+
- spec/cases/parallel_map.rb
|
32
|
+
- spec/cases/parallel_map_sleeping.rb
|
31
33
|
- spec/cases/parallel_sleeping_2.rb
|
32
34
|
- spec/cases/parallel_start_and_kill.rb
|
33
35
|
- spec/cases/parallel_with_detected_cpus.rb
|
@@ -36,7 +38,6 @@ files:
|
|
36
38
|
- spec/spec_helper.rb
|
37
39
|
has_rdoc: false
|
38
40
|
homepage: http://github.com/grosser/parallel
|
39
|
-
licenses:
|
40
41
|
post_install_message:
|
41
42
|
rdoc_options:
|
42
43
|
- --charset=UTF-8
|
@@ -57,7 +58,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
57
58
|
requirements: []
|
58
59
|
|
59
60
|
rubyforge_project:
|
60
|
-
rubygems_version: 1.
|
61
|
+
rubygems_version: 1.2.0
|
61
62
|
signing_key:
|
62
63
|
specification_version: 3
|
63
64
|
summary: Run any kind of code in parallel processes
|
@@ -68,4 +69,6 @@ test_files:
|
|
68
69
|
- spec/cases/parallel_start_and_kill.rb
|
69
70
|
- spec/cases/parallel_with_set_processes.rb
|
70
71
|
- spec/cases/parallel_influence_outside_data.rb
|
72
|
+
- spec/cases/parallel_map_sleeping.rb
|
71
73
|
- spec/cases/parallel_with_detected_cpus.rb
|
74
|
+
- spec/cases/parallel_map.rb
|