lowang-parallel 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +10 -0
- data/README.markdown +52 -0
- data/Rakefile +19 -0
- data/VERSION +1 -0
- data/lib/parallel.rb +166 -0
- data/lowang-parallel.gemspec +82 -0
- data/parallel.gemspec +80 -0
- data/spec/cases/each_with_index.rb +5 -0
- data/spec/cases/map_with_index.rb +6 -0
- data/spec/cases/map_with_index_empty.rb +6 -0
- data/spec/cases/no_dump_with_each.rb +16 -0
- data/spec/cases/parallel_each.rb +9 -0
- data/spec/cases/parallel_high_fork_rate.rb +5 -0
- data/spec/cases/parallel_influence_outside_data.rb +8 -0
- data/spec/cases/parallel_map.rb +6 -0
- data/spec/cases/parallel_map_range.rb +6 -0
- data/spec/cases/parallel_map_sleeping.rb +5 -0
- data/spec/cases/parallel_map_uneven.rb +5 -0
- data/spec/cases/parallel_raise.rb +10 -0
- data/spec/cases/parallel_sleeping_2.rb +5 -0
- data/spec/cases/parallel_start_and_kill.rb +6 -0
- data/spec/cases/parallel_with_detected_cpus.rb +6 -0
- data/spec/cases/parallel_with_nil_uses_detected_cpus.rb +6 -0
- data/spec/cases/parallel_with_set_processes.rb +6 -0
- data/spec/parallel_spec.rb +148 -0
- data/spec/spec_helper.rb +2 -0
- metadata +106 -0
data/.gitignore
ADDED
data/README.markdown
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
Run any code in parallel Processes(> use all CPUs) or Threads(> speedup blocking operations).
|
2
|
+
Best suited for map-reduce or e.g. parallel downloads/uploads.
|
3
|
+
|
4
|
+
Install
|
5
|
+
=======
|
6
|
+
sudo gem install parallel
|
7
|
+
|
8
|
+
Usage
|
9
|
+
=====
|
10
|
+
# 2 CPUs -> work in 2 processes (a,b + c)
|
11
|
+
results = Parallel.map(['a','b','c']) do |one_letter|
|
12
|
+
expensive_calculation(letter)
|
13
|
+
end
|
14
|
+
|
15
|
+
# 3 Processes -> finished after 1 run
|
16
|
+
results = Parallel.map(['a','b','c'], :in_processes=>3){|one_letter| ... }
|
17
|
+
|
18
|
+
# 3 Threads -> finished after 1 run
|
19
|
+
results = Parallel.map(['a','b','c'], :in_threads=>3){|one_letter| ... }
|
20
|
+
|
21
|
+
Same can be done with `each`
|
22
|
+
Parallel.each(['a','b','c']){|one_letter| ... }
|
23
|
+
or `each_with_index` or `map_with_index`
|
24
|
+
|
25
|
+
### Processes
|
26
|
+
- Speedup through multiple CPUs
|
27
|
+
- Speedup for blocking operations
|
28
|
+
- Protects global data
|
29
|
+
- Extra memory used ( very low on [REE](http://www.rubyenterpriseedition.com/faq.html) through `copy_on_write_friendly` )
|
30
|
+
- Child processes are killed when your main process is killed through Ctrl+c or kill -2
|
31
|
+
|
32
|
+
### Threads
|
33
|
+
- Speedup for blocking operations
|
34
|
+
- Global data can be modified
|
35
|
+
- No extra memory used
|
36
|
+
|
37
|
+
|
38
|
+
Processes/Threads are workers, they grab the next piece of work when they finish
|
39
|
+
|
40
|
+
TODO
|
41
|
+
====
|
42
|
+
- JRuby / Windows support <-> possible ?
|
43
|
+
|
44
|
+
Authors
|
45
|
+
=======
|
46
|
+
|
47
|
+
###Contributors (alphabetical)
|
48
|
+
- [TJ Holowaychuk](http://vision-media.ca/) -- tj<$at$>vision-media.ca
|
49
|
+
|
50
|
+
[Michael Grosser](http://pragmatig.wordpress.com)
|
51
|
+
grosser.michael@gmail.com
|
52
|
+
Hereby placed under public domain, do what you want, just do not hold me accountable...
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
task :default => :spec
|
2
|
+
require 'spec/rake/spectask'
|
3
|
+
Spec::Rake::SpecTask.new {|t| t.spec_opts = ['--color']}
|
4
|
+
|
5
|
+
begin
|
6
|
+
require 'jeweler'
|
7
|
+
project_name = 'lowang-parallel'
|
8
|
+
Jeweler::Tasks.new do |gem|
|
9
|
+
gem.name = project_name
|
10
|
+
gem.summary = "Run any kind of code in parallel processes"
|
11
|
+
gem.email = "przemyslaw.wroblewski@gmail.com"
|
12
|
+
gem.homepage = "http://github.com/lowang/#{project_name}"
|
13
|
+
gem.authors = ["Michael Grosser", "Przemyslaw Wroblewski"]
|
14
|
+
end
|
15
|
+
|
16
|
+
Jeweler::GemcutterTasks.new
|
17
|
+
rescue LoadError
|
18
|
+
puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
19
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.4.2
|
data/lib/parallel.rb
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'thread' # to get Thread.exclusive
|
2
|
+
|
3
|
+
class Parallel
|
4
|
+
VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
|
5
|
+
|
6
|
+
def self.in_threads(options={:count => 2})
|
7
|
+
count, options = extract_count_from_options(options)
|
8
|
+
|
9
|
+
out = []
|
10
|
+
threads = []
|
11
|
+
|
12
|
+
count.times do |i|
|
13
|
+
threads[i] = Thread.new do
|
14
|
+
out[i] = yield(i)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
threads.each{|t| t.join }
|
19
|
+
out
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.in_processes(options = {}, &block)
|
23
|
+
count, options = extract_count_from_options(options)
|
24
|
+
count ||= processor_count
|
25
|
+
preserve_results = (options[:preserve_results] != false)
|
26
|
+
|
27
|
+
pipes, pids = fork_and_start_writing(count, :preserve_results => preserve_results, &block)
|
28
|
+
out = read_from_pipes(pipes)
|
29
|
+
pids.each { |pid| Process.wait(pid) }
|
30
|
+
out.map{|x| deserialize(x) } if preserve_results
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.each(array, options={}, &block)
|
34
|
+
map(array, options.merge(:preserve_results => false), &block)
|
35
|
+
array
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.each_with_index(array, options={}, &block)
|
39
|
+
each(array, options.merge(:with_index => true), &block)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.map(array, options = {})
|
43
|
+
array = array.to_a if array.is_a?(Range)
|
44
|
+
|
45
|
+
if options[:in_threads]
|
46
|
+
method = :in_threads
|
47
|
+
size = options[method]
|
48
|
+
else
|
49
|
+
method = :in_processes
|
50
|
+
size = options[method] || processor_count
|
51
|
+
end
|
52
|
+
|
53
|
+
# work in #{size} threads that use threads/processes
|
54
|
+
results = []
|
55
|
+
current = -1
|
56
|
+
|
57
|
+
in_threads(size) do
|
58
|
+
# as long as there are more items, work on one of them
|
59
|
+
loop do
|
60
|
+
index = Thread.exclusive{ current+=1 }
|
61
|
+
break if index >= array.size
|
62
|
+
results[index] = *send(method, options.merge(:count => 1)) do
|
63
|
+
args = [array[index]]
|
64
|
+
args << index if options[:with_index]
|
65
|
+
yield *args
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
results
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.map_with_index(array, options={}, &block)
|
74
|
+
map(array, options.merge(:with_index => true), &block)
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.processor_count
|
78
|
+
case RUBY_PLATFORM
|
79
|
+
when /darwin/
|
80
|
+
`hwprefs cpu_count`.to_i
|
81
|
+
when /linux/
|
82
|
+
`cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
# Collect results from pipes simultanously
|
89
|
+
# otherwise pipes get stuck when to much is written (buffer full)
|
90
|
+
def self.read_from_pipes(reads)
|
91
|
+
out = []
|
92
|
+
in_threads(reads.size) do |i|
|
93
|
+
out[i] = ''
|
94
|
+
while text = reads[i].gets
|
95
|
+
out[i] += text
|
96
|
+
end
|
97
|
+
reads[i].close
|
98
|
+
end
|
99
|
+
out
|
100
|
+
end
|
101
|
+
|
102
|
+
# fork and start writing results into n pipes
|
103
|
+
def self.fork_and_start_writing(count, options, &block)
|
104
|
+
reads = []
|
105
|
+
pids = []
|
106
|
+
count.times do |i|
|
107
|
+
reads[i], write = IO.pipe
|
108
|
+
pids << do_in_new_process(i, options.merge(:write_to => (options[:preserve_results] ? write : nil)), &block)
|
109
|
+
write.close
|
110
|
+
end
|
111
|
+
kill_on_ctrl_c(pids)
|
112
|
+
[reads, pids]
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.do_in_new_process(work_item, options)
|
116
|
+
# activate copy on write friendly GC of REE
|
117
|
+
GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
|
118
|
+
Process.fork do
|
119
|
+
result = yield(work_item)
|
120
|
+
serialize(result, options) if options[:write_to]
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.serialize(something, options)
|
125
|
+
Marshal.dump(something, options[:write_to])
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.deserialize(something)
|
129
|
+
Marshal.load(something)
|
130
|
+
end
|
131
|
+
|
132
|
+
# options is either a Interger or a Hash with :count
|
133
|
+
def self.extract_count_from_options(options)
|
134
|
+
if options.is_a?(Hash)
|
135
|
+
count = options[:count]
|
136
|
+
else
|
137
|
+
count = options
|
138
|
+
options = {}
|
139
|
+
end
|
140
|
+
[count, options]
|
141
|
+
end
|
142
|
+
|
143
|
+
# split an array into groups of size items
|
144
|
+
# (copied from ActiveSupport, to not require it)
|
145
|
+
def self.in_groups_of(array, size)
|
146
|
+
results = []
|
147
|
+
loop do
|
148
|
+
slice = array[(results.size * size)...((results.size+1) * size)]
|
149
|
+
if slice.nil? or slice.empty?
|
150
|
+
break
|
151
|
+
else
|
152
|
+
results << slice
|
153
|
+
end
|
154
|
+
end
|
155
|
+
results
|
156
|
+
end
|
157
|
+
|
158
|
+
# kill all these processes (children) if user presses Ctrl+c
|
159
|
+
def self.kill_on_ctrl_c(pids)
|
160
|
+
Signal.trap :SIGINT do
|
161
|
+
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
162
|
+
pids.each { |pid| Process.kill(:KILL, pid) }
|
163
|
+
exit 1 # Quit with 'failed' signal
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{lowang-parallel}
|
8
|
+
s.version = "0.4.2"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Michael Grosser", "Przemyslaw Wroblewski"]
|
12
|
+
s.date = %q{2010-05-10}
|
13
|
+
s.email = %q{przemyslaw.wroblewski@gmail.com}
|
14
|
+
s.extra_rdoc_files = [
|
15
|
+
"README.markdown"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
".gitignore",
|
19
|
+
"README.markdown",
|
20
|
+
"Rakefile",
|
21
|
+
"VERSION",
|
22
|
+
"lib/parallel.rb",
|
23
|
+
"lowang-parallel.gemspec",
|
24
|
+
"parallel.gemspec",
|
25
|
+
"spec/cases/each_with_index.rb",
|
26
|
+
"spec/cases/map_with_index.rb",
|
27
|
+
"spec/cases/map_with_index_empty.rb",
|
28
|
+
"spec/cases/no_dump_with_each.rb",
|
29
|
+
"spec/cases/parallel_each.rb",
|
30
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
31
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
32
|
+
"spec/cases/parallel_map.rb",
|
33
|
+
"spec/cases/parallel_map_range.rb",
|
34
|
+
"spec/cases/parallel_map_sleeping.rb",
|
35
|
+
"spec/cases/parallel_map_uneven.rb",
|
36
|
+
"spec/cases/parallel_raise.rb",
|
37
|
+
"spec/cases/parallel_sleeping_2.rb",
|
38
|
+
"spec/cases/parallel_start_and_kill.rb",
|
39
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
40
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
41
|
+
"spec/cases/parallel_with_set_processes.rb",
|
42
|
+
"spec/parallel_spec.rb",
|
43
|
+
"spec/spec_helper.rb"
|
44
|
+
]
|
45
|
+
s.homepage = %q{http://github.com/lowang/lowang-parallel}
|
46
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
47
|
+
s.require_paths = ["lib"]
|
48
|
+
s.rubygems_version = %q{1.3.6}
|
49
|
+
s.summary = %q{Run any kind of code in parallel processes}
|
50
|
+
s.test_files = [
|
51
|
+
"spec/cases/each_with_index.rb",
|
52
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
53
|
+
"spec/cases/parallel_raise.rb",
|
54
|
+
"spec/cases/parallel_map_uneven.rb",
|
55
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
56
|
+
"spec/cases/map_with_index_empty.rb",
|
57
|
+
"spec/cases/parallel_map_range.rb",
|
58
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
59
|
+
"spec/cases/map_with_index.rb",
|
60
|
+
"spec/cases/parallel_start_and_kill.rb",
|
61
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
62
|
+
"spec/cases/no_dump_with_each.rb",
|
63
|
+
"spec/cases/parallel_map.rb",
|
64
|
+
"spec/cases/parallel_each.rb",
|
65
|
+
"spec/cases/parallel_sleeping_2.rb",
|
66
|
+
"spec/cases/parallel_with_set_processes.rb",
|
67
|
+
"spec/cases/parallel_map_sleeping.rb",
|
68
|
+
"spec/parallel_spec.rb",
|
69
|
+
"spec/spec_helper.rb"
|
70
|
+
]
|
71
|
+
|
72
|
+
if s.respond_to? :specification_version then
|
73
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
74
|
+
s.specification_version = 3
|
75
|
+
|
76
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
77
|
+
else
|
78
|
+
end
|
79
|
+
else
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
data/parallel.gemspec
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{parallel}
|
8
|
+
s.version = "0.4.1"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Michael Grosser"]
|
12
|
+
s.date = %q{2010-04-18}
|
13
|
+
s.email = %q{grosser.michael@gmail.com}
|
14
|
+
s.extra_rdoc_files = [
|
15
|
+
"README.markdown"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
"README.markdown",
|
19
|
+
"Rakefile",
|
20
|
+
"VERSION",
|
21
|
+
"lib/parallel.rb",
|
22
|
+
"parallel.gemspec",
|
23
|
+
"spec/cases/each_with_index.rb",
|
24
|
+
"spec/cases/map_with_index.rb",
|
25
|
+
"spec/cases/map_with_index_empty.rb",
|
26
|
+
"spec/cases/no_dump_with_each.rb",
|
27
|
+
"spec/cases/parallel_each.rb",
|
28
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
29
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
30
|
+
"spec/cases/parallel_map.rb",
|
31
|
+
"spec/cases/parallel_map_range.rb",
|
32
|
+
"spec/cases/parallel_map_sleeping.rb",
|
33
|
+
"spec/cases/parallel_map_uneven.rb",
|
34
|
+
"spec/cases/parallel_raise.rb",
|
35
|
+
"spec/cases/parallel_sleeping_2.rb",
|
36
|
+
"spec/cases/parallel_start_and_kill.rb",
|
37
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
38
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
39
|
+
"spec/cases/parallel_with_set_processes.rb",
|
40
|
+
"spec/parallel_spec.rb",
|
41
|
+
"spec/spec_helper.rb"
|
42
|
+
]
|
43
|
+
s.homepage = %q{http://github.com/grosser/parallel}
|
44
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
45
|
+
s.require_paths = ["lib"]
|
46
|
+
s.rubygems_version = %q{1.3.6}
|
47
|
+
s.summary = %q{Run any kind of code in parallel processes}
|
48
|
+
s.test_files = [
|
49
|
+
"spec/spec_helper.rb",
|
50
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
51
|
+
"spec/cases/map_with_index_empty.rb",
|
52
|
+
"spec/cases/parallel_map_uneven.rb",
|
53
|
+
"spec/cases/parallel_map_range.rb",
|
54
|
+
"spec/cases/map_with_index.rb",
|
55
|
+
"spec/cases/parallel_with_set_processes.rb",
|
56
|
+
"spec/cases/no_dump_with_each.rb",
|
57
|
+
"spec/cases/each_with_index.rb",
|
58
|
+
"spec/cases/parallel_start_and_kill.rb",
|
59
|
+
"spec/cases/parallel_raise.rb",
|
60
|
+
"spec/cases/parallel_sleeping_2.rb",
|
61
|
+
"spec/cases/parallel_each.rb",
|
62
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
63
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
64
|
+
"spec/cases/parallel_map.rb",
|
65
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
66
|
+
"spec/cases/parallel_map_sleeping.rb",
|
67
|
+
"spec/parallel_spec.rb"
|
68
|
+
]
|
69
|
+
|
70
|
+
if s.respond_to? :specification_version then
|
71
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
72
|
+
s.specification_version = 3
|
73
|
+
|
74
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
75
|
+
else
|
76
|
+
end
|
77
|
+
else
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
@@ -0,0 +1,148 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe Parallel do
|
4
|
+
describe :in_processes do
|
5
|
+
before do
|
6
|
+
@cpus = Parallel.processor_count
|
7
|
+
end
|
8
|
+
|
9
|
+
it "executes with detected cpus" do
|
10
|
+
`ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * @cpus
|
11
|
+
end
|
12
|
+
|
13
|
+
it "executes with detected cpus when nil was given" do
|
14
|
+
`ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * @cpus
|
15
|
+
end
|
16
|
+
|
17
|
+
it "set ammount of parallel processes" do
|
18
|
+
`ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
|
19
|
+
end
|
20
|
+
|
21
|
+
it "does not influence outside data" do
|
22
|
+
`ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "kills the processes when the main process gets killed through ctrl+c" do
|
26
|
+
t = Time.now
|
27
|
+
lambda{
|
28
|
+
Thread.new do
|
29
|
+
`ruby spec/cases/parallel_start_and_kill.rb`
|
30
|
+
end
|
31
|
+
sleep 1
|
32
|
+
running_processes = `ps -f`.split("\n").map{|line| line.split(/\s+/)}
|
33
|
+
parent = running_processes.detect{|line| line.include?("00:00:00") and line.include?("ruby") }[1]
|
34
|
+
`kill -2 #{parent}` #simulates Ctrl+c
|
35
|
+
}.should_not change{`ps`.split("\n").size}
|
36
|
+
Time.now.should be_close(t, 3)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "saves time" do
|
40
|
+
t = Time.now
|
41
|
+
`ruby spec/cases/parallel_sleeping_2.rb`
|
42
|
+
Time.now.should be_close(t, 3)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "raises when one of the processes raises" do
|
46
|
+
pending 'there is some kind of error, but not the original...'
|
47
|
+
`ruby spec/cases/parallel_raise.rb`.should == 'TEST'
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'can handle to high fork rate' do
|
51
|
+
`ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe :in_threads do
|
56
|
+
it "saves time" do
|
57
|
+
t = Time.now
|
58
|
+
Parallel.in_threads(3){ sleep 2 }
|
59
|
+
Time.now.should be_close(t, 3)
|
60
|
+
end
|
61
|
+
|
62
|
+
it "does not create new processes" do
|
63
|
+
lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
|
64
|
+
end
|
65
|
+
|
66
|
+
it "returns results as array" do
|
67
|
+
Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
|
68
|
+
end
|
69
|
+
|
70
|
+
it "raises when a thread raises" do
|
71
|
+
lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe :map do
|
76
|
+
it "saves time" do
|
77
|
+
t = Time.now
|
78
|
+
`ruby spec/cases/parallel_map_sleeping.rb`
|
79
|
+
Time.now.should be_close(t, 3)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "executes with given parameters" do
|
83
|
+
`ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
|
84
|
+
end
|
85
|
+
|
86
|
+
it "starts new process imediatly when old exists" do
|
87
|
+
t = Time.now
|
88
|
+
`ruby spec/cases/parallel_map_uneven.rb`
|
89
|
+
Time.now.should be_close(t, 3)
|
90
|
+
end
|
91
|
+
|
92
|
+
it "does not flatten results" do
|
93
|
+
Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
|
94
|
+
end
|
95
|
+
|
96
|
+
it "can run in threads" do
|
97
|
+
Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'supports ranges' do
|
101
|
+
`ruby spec/cases/parallel_map_range.rb`.should == '12345'
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
describe :map_with_index do
|
106
|
+
it "yields object and index" do
|
107
|
+
`ruby spec/cases/map_with_index.rb 2>&1`.should == 'a0b1'
|
108
|
+
end
|
109
|
+
|
110
|
+
it "does not crash with empty set" do
|
111
|
+
`ruby spec/cases/map_with_index_empty.rb 2>&1`.should == ''
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
describe :each do
|
116
|
+
it "returns original array, works like map" do
|
117
|
+
`ruby spec/cases/parallel_each.rb`.should == '-b--c--d--a-a b c d'
|
118
|
+
end
|
119
|
+
|
120
|
+
it "does not use marshal_dump" do
|
121
|
+
`ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'not dumpable'
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
describe :each_with_index do
|
126
|
+
it "yields object and index" do
|
127
|
+
`ruby spec/cases/each_with_index.rb 2>&1`.should == 'a0b1'
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
describe :in_groups_of do
|
132
|
+
it "works for empty" do
|
133
|
+
Parallel.send(:in_groups_of, [], 3).should == []
|
134
|
+
end
|
135
|
+
|
136
|
+
it "works for smaller then count" do
|
137
|
+
Parallel.send(:in_groups_of, [1,2], 3).should == [[1,2]]
|
138
|
+
end
|
139
|
+
|
140
|
+
it "works for count" do
|
141
|
+
Parallel.send(:in_groups_of, [1,2,3], 3).should == [[1,2,3]]
|
142
|
+
end
|
143
|
+
|
144
|
+
it "works for larger than count" do
|
145
|
+
Parallel.send(:in_groups_of, [1,2,3,4], 3).should == [[1,2,3],[4]]
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: lowang-parallel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 4
|
8
|
+
- 2
|
9
|
+
version: 0.4.2
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Michael Grosser
|
13
|
+
- Przemyslaw Wroblewski
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-05-10 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description:
|
23
|
+
email: przemyslaw.wroblewski@gmail.com
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files:
|
29
|
+
- README.markdown
|
30
|
+
files:
|
31
|
+
- .gitignore
|
32
|
+
- README.markdown
|
33
|
+
- Rakefile
|
34
|
+
- VERSION
|
35
|
+
- lib/parallel.rb
|
36
|
+
- lowang-parallel.gemspec
|
37
|
+
- parallel.gemspec
|
38
|
+
- spec/cases/each_with_index.rb
|
39
|
+
- spec/cases/map_with_index.rb
|
40
|
+
- spec/cases/map_with_index_empty.rb
|
41
|
+
- spec/cases/no_dump_with_each.rb
|
42
|
+
- spec/cases/parallel_each.rb
|
43
|
+
- spec/cases/parallel_high_fork_rate.rb
|
44
|
+
- spec/cases/parallel_influence_outside_data.rb
|
45
|
+
- spec/cases/parallel_map.rb
|
46
|
+
- spec/cases/parallel_map_range.rb
|
47
|
+
- spec/cases/parallel_map_sleeping.rb
|
48
|
+
- spec/cases/parallel_map_uneven.rb
|
49
|
+
- spec/cases/parallel_raise.rb
|
50
|
+
- spec/cases/parallel_sleeping_2.rb
|
51
|
+
- spec/cases/parallel_start_and_kill.rb
|
52
|
+
- spec/cases/parallel_with_detected_cpus.rb
|
53
|
+
- spec/cases/parallel_with_nil_uses_detected_cpus.rb
|
54
|
+
- spec/cases/parallel_with_set_processes.rb
|
55
|
+
- spec/parallel_spec.rb
|
56
|
+
- spec/spec_helper.rb
|
57
|
+
has_rdoc: true
|
58
|
+
homepage: http://github.com/lowang/lowang-parallel
|
59
|
+
licenses: []
|
60
|
+
|
61
|
+
post_install_message:
|
62
|
+
rdoc_options:
|
63
|
+
- --charset=UTF-8
|
64
|
+
require_paths:
|
65
|
+
- lib
|
66
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
segments:
|
71
|
+
- 0
|
72
|
+
version: "0"
|
73
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
segments:
|
78
|
+
- 0
|
79
|
+
version: "0"
|
80
|
+
requirements: []
|
81
|
+
|
82
|
+
rubyforge_project:
|
83
|
+
rubygems_version: 1.3.6
|
84
|
+
signing_key:
|
85
|
+
specification_version: 3
|
86
|
+
summary: Run any kind of code in parallel processes
|
87
|
+
test_files:
|
88
|
+
- spec/cases/each_with_index.rb
|
89
|
+
- spec/cases/parallel_with_nil_uses_detected_cpus.rb
|
90
|
+
- spec/cases/parallel_raise.rb
|
91
|
+
- spec/cases/parallel_map_uneven.rb
|
92
|
+
- spec/cases/parallel_high_fork_rate.rb
|
93
|
+
- spec/cases/map_with_index_empty.rb
|
94
|
+
- spec/cases/parallel_map_range.rb
|
95
|
+
- spec/cases/parallel_influence_outside_data.rb
|
96
|
+
- spec/cases/map_with_index.rb
|
97
|
+
- spec/cases/parallel_start_and_kill.rb
|
98
|
+
- spec/cases/parallel_with_detected_cpus.rb
|
99
|
+
- spec/cases/no_dump_with_each.rb
|
100
|
+
- spec/cases/parallel_map.rb
|
101
|
+
- spec/cases/parallel_each.rb
|
102
|
+
- spec/cases/parallel_sleeping_2.rb
|
103
|
+
- spec/cases/parallel_with_set_processes.rb
|
104
|
+
- spec/cases/parallel_map_sleeping.rb
|
105
|
+
- spec/parallel_spec.rb
|
106
|
+
- spec/spec_helper.rb
|