lowang-parallel 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +10 -0
- data/README.markdown +52 -0
- data/Rakefile +19 -0
- data/VERSION +1 -0
- data/lib/parallel.rb +166 -0
- data/lowang-parallel.gemspec +82 -0
- data/parallel.gemspec +80 -0
- data/spec/cases/each_with_index.rb +5 -0
- data/spec/cases/map_with_index.rb +6 -0
- data/spec/cases/map_with_index_empty.rb +6 -0
- data/spec/cases/no_dump_with_each.rb +16 -0
- data/spec/cases/parallel_each.rb +9 -0
- data/spec/cases/parallel_high_fork_rate.rb +5 -0
- data/spec/cases/parallel_influence_outside_data.rb +8 -0
- data/spec/cases/parallel_map.rb +6 -0
- data/spec/cases/parallel_map_range.rb +6 -0
- data/spec/cases/parallel_map_sleeping.rb +5 -0
- data/spec/cases/parallel_map_uneven.rb +5 -0
- data/spec/cases/parallel_raise.rb +10 -0
- data/spec/cases/parallel_sleeping_2.rb +5 -0
- data/spec/cases/parallel_start_and_kill.rb +6 -0
- data/spec/cases/parallel_with_detected_cpus.rb +6 -0
- data/spec/cases/parallel_with_nil_uses_detected_cpus.rb +6 -0
- data/spec/cases/parallel_with_set_processes.rb +6 -0
- data/spec/parallel_spec.rb +148 -0
- data/spec/spec_helper.rb +2 -0
- metadata +106 -0
data/.gitignore
ADDED
data/README.markdown
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
Run any code in parallel Processes(> use all CPUs) or Threads(> speedup blocking operations).
|
2
|
+
Best suited for map-reduce or e.g. parallel downloads/uploads.
|
3
|
+
|
4
|
+
Install
|
5
|
+
=======
|
6
|
+
sudo gem install parallel
|
7
|
+
|
8
|
+
Usage
|
9
|
+
=====
|
10
|
+
# 2 CPUs -> work in 2 processes (a,b + c)
|
11
|
+
results = Parallel.map(['a','b','c']) do |one_letter|
|
12
|
+
expensive_calculation(letter)
|
13
|
+
end
|
14
|
+
|
15
|
+
# 3 Processes -> finished after 1 run
|
16
|
+
results = Parallel.map(['a','b','c'], :in_processes=>3){|one_letter| ... }
|
17
|
+
|
18
|
+
# 3 Threads -> finished after 1 run
|
19
|
+
results = Parallel.map(['a','b','c'], :in_threads=>3){|one_letter| ... }
|
20
|
+
|
21
|
+
Same can be done with `each`
|
22
|
+
Parallel.each(['a','b','c']){|one_letter| ... }
|
23
|
+
or `each_with_index` or `map_with_index`
|
24
|
+
|
25
|
+
### Processes
|
26
|
+
- Speedup through multiple CPUs
|
27
|
+
- Speedup for blocking operations
|
28
|
+
- Protects global data
|
29
|
+
- Extra memory used ( very low on [REE](http://www.rubyenterpriseedition.com/faq.html) through `copy_on_write_friendly` )
|
30
|
+
- Child processes are killed when your main process is killed through Ctrl+c or kill -2
|
31
|
+
|
32
|
+
### Threads
|
33
|
+
- Speedup for blocking operations
|
34
|
+
- Global data can be modified
|
35
|
+
- No extra memory used
|
36
|
+
|
37
|
+
|
38
|
+
Processes/Threads are workers, they grab the next piece of work when they finish
|
39
|
+
|
40
|
+
TODO
|
41
|
+
====
|
42
|
+
- JRuby / Windows support <-> possible ?
|
43
|
+
|
44
|
+
Authors
|
45
|
+
=======
|
46
|
+
|
47
|
+
###Contributors (alphabetical)
|
48
|
+
- [TJ Holowaychuk](http://vision-media.ca/) -- tj<$at$>vision-media.ca
|
49
|
+
|
50
|
+
[Michael Grosser](http://pragmatig.wordpress.com)
|
51
|
+
grosser.michael@gmail.com
|
52
|
+
Hereby placed under public domain, do what you want, just do not hold me accountable...
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
task :default => :spec
|
2
|
+
require 'spec/rake/spectask'
|
3
|
+
Spec::Rake::SpecTask.new {|t| t.spec_opts = ['--color']}
|
4
|
+
|
5
|
+
begin
|
6
|
+
require 'jeweler'
|
7
|
+
project_name = 'lowang-parallel'
|
8
|
+
Jeweler::Tasks.new do |gem|
|
9
|
+
gem.name = project_name
|
10
|
+
gem.summary = "Run any kind of code in parallel processes"
|
11
|
+
gem.email = "przemyslaw.wroblewski@gmail.com"
|
12
|
+
gem.homepage = "http://github.com/lowang/#{project_name}"
|
13
|
+
gem.authors = ["Michael Grosser", "Przemyslaw Wroblewski"]
|
14
|
+
end
|
15
|
+
|
16
|
+
Jeweler::GemcutterTasks.new
|
17
|
+
rescue LoadError
|
18
|
+
puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
19
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.4.2
|
data/lib/parallel.rb
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'thread' # to get Thread.exclusive
|
2
|
+
|
3
|
+
class Parallel
|
4
|
+
VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
|
5
|
+
|
6
|
+
def self.in_threads(options={:count => 2})
|
7
|
+
count, options = extract_count_from_options(options)
|
8
|
+
|
9
|
+
out = []
|
10
|
+
threads = []
|
11
|
+
|
12
|
+
count.times do |i|
|
13
|
+
threads[i] = Thread.new do
|
14
|
+
out[i] = yield(i)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
threads.each{|t| t.join }
|
19
|
+
out
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.in_processes(options = {}, &block)
|
23
|
+
count, options = extract_count_from_options(options)
|
24
|
+
count ||= processor_count
|
25
|
+
preserve_results = (options[:preserve_results] != false)
|
26
|
+
|
27
|
+
pipes, pids = fork_and_start_writing(count, :preserve_results => preserve_results, &block)
|
28
|
+
out = read_from_pipes(pipes)
|
29
|
+
pids.each { |pid| Process.wait(pid) }
|
30
|
+
out.map{|x| deserialize(x) } if preserve_results
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.each(array, options={}, &block)
|
34
|
+
map(array, options.merge(:preserve_results => false), &block)
|
35
|
+
array
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.each_with_index(array, options={}, &block)
|
39
|
+
each(array, options.merge(:with_index => true), &block)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.map(array, options = {})
|
43
|
+
array = array.to_a if array.is_a?(Range)
|
44
|
+
|
45
|
+
if options[:in_threads]
|
46
|
+
method = :in_threads
|
47
|
+
size = options[method]
|
48
|
+
else
|
49
|
+
method = :in_processes
|
50
|
+
size = options[method] || processor_count
|
51
|
+
end
|
52
|
+
|
53
|
+
# work in #{size} threads that use threads/processes
|
54
|
+
results = []
|
55
|
+
current = -1
|
56
|
+
|
57
|
+
in_threads(size) do
|
58
|
+
# as long as there are more items, work on one of them
|
59
|
+
loop do
|
60
|
+
index = Thread.exclusive{ current+=1 }
|
61
|
+
break if index >= array.size
|
62
|
+
results[index] = *send(method, options.merge(:count => 1)) do
|
63
|
+
args = [array[index]]
|
64
|
+
args << index if options[:with_index]
|
65
|
+
yield *args
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
results
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.map_with_index(array, options={}, &block)
|
74
|
+
map(array, options.merge(:with_index => true), &block)
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.processor_count
|
78
|
+
case RUBY_PLATFORM
|
79
|
+
when /darwin/
|
80
|
+
`hwprefs cpu_count`.to_i
|
81
|
+
when /linux/
|
82
|
+
`cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
# Collect results from pipes simultanously
|
89
|
+
# otherwise pipes get stuck when to much is written (buffer full)
|
90
|
+
def self.read_from_pipes(reads)
|
91
|
+
out = []
|
92
|
+
in_threads(reads.size) do |i|
|
93
|
+
out[i] = ''
|
94
|
+
while text = reads[i].gets
|
95
|
+
out[i] += text
|
96
|
+
end
|
97
|
+
reads[i].close
|
98
|
+
end
|
99
|
+
out
|
100
|
+
end
|
101
|
+
|
102
|
+
# fork and start writing results into n pipes
|
103
|
+
def self.fork_and_start_writing(count, options, &block)
|
104
|
+
reads = []
|
105
|
+
pids = []
|
106
|
+
count.times do |i|
|
107
|
+
reads[i], write = IO.pipe
|
108
|
+
pids << do_in_new_process(i, options.merge(:write_to => (options[:preserve_results] ? write : nil)), &block)
|
109
|
+
write.close
|
110
|
+
end
|
111
|
+
kill_on_ctrl_c(pids)
|
112
|
+
[reads, pids]
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.do_in_new_process(work_item, options)
|
116
|
+
# activate copy on write friendly GC of REE
|
117
|
+
GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
|
118
|
+
Process.fork do
|
119
|
+
result = yield(work_item)
|
120
|
+
serialize(result, options) if options[:write_to]
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.serialize(something, options)
|
125
|
+
Marshal.dump(something, options[:write_to])
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.deserialize(something)
|
129
|
+
Marshal.load(something)
|
130
|
+
end
|
131
|
+
|
132
|
+
# options is either a Interger or a Hash with :count
|
133
|
+
def self.extract_count_from_options(options)
|
134
|
+
if options.is_a?(Hash)
|
135
|
+
count = options[:count]
|
136
|
+
else
|
137
|
+
count = options
|
138
|
+
options = {}
|
139
|
+
end
|
140
|
+
[count, options]
|
141
|
+
end
|
142
|
+
|
143
|
+
# split an array into groups of size items
|
144
|
+
# (copied from ActiveSupport, to not require it)
|
145
|
+
def self.in_groups_of(array, size)
|
146
|
+
results = []
|
147
|
+
loop do
|
148
|
+
slice = array[(results.size * size)...((results.size+1) * size)]
|
149
|
+
if slice.nil? or slice.empty?
|
150
|
+
break
|
151
|
+
else
|
152
|
+
results << slice
|
153
|
+
end
|
154
|
+
end
|
155
|
+
results
|
156
|
+
end
|
157
|
+
|
158
|
+
# kill all these processes (children) if user presses Ctrl+c
|
159
|
+
def self.kill_on_ctrl_c(pids)
|
160
|
+
Signal.trap :SIGINT do
|
161
|
+
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
162
|
+
pids.each { |pid| Process.kill(:KILL, pid) }
|
163
|
+
exit 1 # Quit with 'failed' signal
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{lowang-parallel}
|
8
|
+
s.version = "0.4.2"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Michael Grosser", "Przemyslaw Wroblewski"]
|
12
|
+
s.date = %q{2010-05-10}
|
13
|
+
s.email = %q{przemyslaw.wroblewski@gmail.com}
|
14
|
+
s.extra_rdoc_files = [
|
15
|
+
"README.markdown"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
".gitignore",
|
19
|
+
"README.markdown",
|
20
|
+
"Rakefile",
|
21
|
+
"VERSION",
|
22
|
+
"lib/parallel.rb",
|
23
|
+
"lowang-parallel.gemspec",
|
24
|
+
"parallel.gemspec",
|
25
|
+
"spec/cases/each_with_index.rb",
|
26
|
+
"spec/cases/map_with_index.rb",
|
27
|
+
"spec/cases/map_with_index_empty.rb",
|
28
|
+
"spec/cases/no_dump_with_each.rb",
|
29
|
+
"spec/cases/parallel_each.rb",
|
30
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
31
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
32
|
+
"spec/cases/parallel_map.rb",
|
33
|
+
"spec/cases/parallel_map_range.rb",
|
34
|
+
"spec/cases/parallel_map_sleeping.rb",
|
35
|
+
"spec/cases/parallel_map_uneven.rb",
|
36
|
+
"spec/cases/parallel_raise.rb",
|
37
|
+
"spec/cases/parallel_sleeping_2.rb",
|
38
|
+
"spec/cases/parallel_start_and_kill.rb",
|
39
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
40
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
41
|
+
"spec/cases/parallel_with_set_processes.rb",
|
42
|
+
"spec/parallel_spec.rb",
|
43
|
+
"spec/spec_helper.rb"
|
44
|
+
]
|
45
|
+
s.homepage = %q{http://github.com/lowang/lowang-parallel}
|
46
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
47
|
+
s.require_paths = ["lib"]
|
48
|
+
s.rubygems_version = %q{1.3.6}
|
49
|
+
s.summary = %q{Run any kind of code in parallel processes}
|
50
|
+
s.test_files = [
|
51
|
+
"spec/cases/each_with_index.rb",
|
52
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
53
|
+
"spec/cases/parallel_raise.rb",
|
54
|
+
"spec/cases/parallel_map_uneven.rb",
|
55
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
56
|
+
"spec/cases/map_with_index_empty.rb",
|
57
|
+
"spec/cases/parallel_map_range.rb",
|
58
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
59
|
+
"spec/cases/map_with_index.rb",
|
60
|
+
"spec/cases/parallel_start_and_kill.rb",
|
61
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
62
|
+
"spec/cases/no_dump_with_each.rb",
|
63
|
+
"spec/cases/parallel_map.rb",
|
64
|
+
"spec/cases/parallel_each.rb",
|
65
|
+
"spec/cases/parallel_sleeping_2.rb",
|
66
|
+
"spec/cases/parallel_with_set_processes.rb",
|
67
|
+
"spec/cases/parallel_map_sleeping.rb",
|
68
|
+
"spec/parallel_spec.rb",
|
69
|
+
"spec/spec_helper.rb"
|
70
|
+
]
|
71
|
+
|
72
|
+
if s.respond_to? :specification_version then
|
73
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
74
|
+
s.specification_version = 3
|
75
|
+
|
76
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
77
|
+
else
|
78
|
+
end
|
79
|
+
else
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
data/parallel.gemspec
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{parallel}
|
8
|
+
s.version = "0.4.1"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Michael Grosser"]
|
12
|
+
s.date = %q{2010-04-18}
|
13
|
+
s.email = %q{grosser.michael@gmail.com}
|
14
|
+
s.extra_rdoc_files = [
|
15
|
+
"README.markdown"
|
16
|
+
]
|
17
|
+
s.files = [
|
18
|
+
"README.markdown",
|
19
|
+
"Rakefile",
|
20
|
+
"VERSION",
|
21
|
+
"lib/parallel.rb",
|
22
|
+
"parallel.gemspec",
|
23
|
+
"spec/cases/each_with_index.rb",
|
24
|
+
"spec/cases/map_with_index.rb",
|
25
|
+
"spec/cases/map_with_index_empty.rb",
|
26
|
+
"spec/cases/no_dump_with_each.rb",
|
27
|
+
"spec/cases/parallel_each.rb",
|
28
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
29
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
30
|
+
"spec/cases/parallel_map.rb",
|
31
|
+
"spec/cases/parallel_map_range.rb",
|
32
|
+
"spec/cases/parallel_map_sleeping.rb",
|
33
|
+
"spec/cases/parallel_map_uneven.rb",
|
34
|
+
"spec/cases/parallel_raise.rb",
|
35
|
+
"spec/cases/parallel_sleeping_2.rb",
|
36
|
+
"spec/cases/parallel_start_and_kill.rb",
|
37
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
38
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
39
|
+
"spec/cases/parallel_with_set_processes.rb",
|
40
|
+
"spec/parallel_spec.rb",
|
41
|
+
"spec/spec_helper.rb"
|
42
|
+
]
|
43
|
+
s.homepage = %q{http://github.com/grosser/parallel}
|
44
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
45
|
+
s.require_paths = ["lib"]
|
46
|
+
s.rubygems_version = %q{1.3.6}
|
47
|
+
s.summary = %q{Run any kind of code in parallel processes}
|
48
|
+
s.test_files = [
|
49
|
+
"spec/spec_helper.rb",
|
50
|
+
"spec/cases/parallel_with_nil_uses_detected_cpus.rb",
|
51
|
+
"spec/cases/map_with_index_empty.rb",
|
52
|
+
"spec/cases/parallel_map_uneven.rb",
|
53
|
+
"spec/cases/parallel_map_range.rb",
|
54
|
+
"spec/cases/map_with_index.rb",
|
55
|
+
"spec/cases/parallel_with_set_processes.rb",
|
56
|
+
"spec/cases/no_dump_with_each.rb",
|
57
|
+
"spec/cases/each_with_index.rb",
|
58
|
+
"spec/cases/parallel_start_and_kill.rb",
|
59
|
+
"spec/cases/parallel_raise.rb",
|
60
|
+
"spec/cases/parallel_sleeping_2.rb",
|
61
|
+
"spec/cases/parallel_each.rb",
|
62
|
+
"spec/cases/parallel_influence_outside_data.rb",
|
63
|
+
"spec/cases/parallel_high_fork_rate.rb",
|
64
|
+
"spec/cases/parallel_map.rb",
|
65
|
+
"spec/cases/parallel_with_detected_cpus.rb",
|
66
|
+
"spec/cases/parallel_map_sleeping.rb",
|
67
|
+
"spec/parallel_spec.rb"
|
68
|
+
]
|
69
|
+
|
70
|
+
if s.respond_to? :specification_version then
|
71
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
72
|
+
s.specification_version = 3
|
73
|
+
|
74
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
75
|
+
else
|
76
|
+
end
|
77
|
+
else
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
@@ -0,0 +1,148 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe Parallel do
|
4
|
+
describe :in_processes do
|
5
|
+
before do
|
6
|
+
@cpus = Parallel.processor_count
|
7
|
+
end
|
8
|
+
|
9
|
+
it "executes with detected cpus" do
|
10
|
+
`ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * @cpus
|
11
|
+
end
|
12
|
+
|
13
|
+
it "executes with detected cpus when nil was given" do
|
14
|
+
`ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * @cpus
|
15
|
+
end
|
16
|
+
|
17
|
+
it "set ammount of parallel processes" do
|
18
|
+
`ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
|
19
|
+
end
|
20
|
+
|
21
|
+
it "does not influence outside data" do
|
22
|
+
`ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "kills the processes when the main process gets killed through ctrl+c" do
|
26
|
+
t = Time.now
|
27
|
+
lambda{
|
28
|
+
Thread.new do
|
29
|
+
`ruby spec/cases/parallel_start_and_kill.rb`
|
30
|
+
end
|
31
|
+
sleep 1
|
32
|
+
running_processes = `ps -f`.split("\n").map{|line| line.split(/\s+/)}
|
33
|
+
parent = running_processes.detect{|line| line.include?("00:00:00") and line.include?("ruby") }[1]
|
34
|
+
`kill -2 #{parent}` #simulates Ctrl+c
|
35
|
+
}.should_not change{`ps`.split("\n").size}
|
36
|
+
Time.now.should be_close(t, 3)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "saves time" do
|
40
|
+
t = Time.now
|
41
|
+
`ruby spec/cases/parallel_sleeping_2.rb`
|
42
|
+
Time.now.should be_close(t, 3)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "raises when one of the processes raises" do
|
46
|
+
pending 'there is some kind of error, but not the original...'
|
47
|
+
`ruby spec/cases/parallel_raise.rb`.should == 'TEST'
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'can handle to high fork rate' do
|
51
|
+
`ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe :in_threads do
|
56
|
+
it "saves time" do
|
57
|
+
t = Time.now
|
58
|
+
Parallel.in_threads(3){ sleep 2 }
|
59
|
+
Time.now.should be_close(t, 3)
|
60
|
+
end
|
61
|
+
|
62
|
+
it "does not create new processes" do
|
63
|
+
lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
|
64
|
+
end
|
65
|
+
|
66
|
+
it "returns results as array" do
|
67
|
+
Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
|
68
|
+
end
|
69
|
+
|
70
|
+
it "raises when a thread raises" do
|
71
|
+
lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe :map do
|
76
|
+
it "saves time" do
|
77
|
+
t = Time.now
|
78
|
+
`ruby spec/cases/parallel_map_sleeping.rb`
|
79
|
+
Time.now.should be_close(t, 3)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "executes with given parameters" do
|
83
|
+
`ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
|
84
|
+
end
|
85
|
+
|
86
|
+
it "starts new process imediatly when old exists" do
|
87
|
+
t = Time.now
|
88
|
+
`ruby spec/cases/parallel_map_uneven.rb`
|
89
|
+
Time.now.should be_close(t, 3)
|
90
|
+
end
|
91
|
+
|
92
|
+
it "does not flatten results" do
|
93
|
+
Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
|
94
|
+
end
|
95
|
+
|
96
|
+
it "can run in threads" do
|
97
|
+
Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'supports ranges' do
|
101
|
+
`ruby spec/cases/parallel_map_range.rb`.should == '12345'
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
describe :map_with_index do
|
106
|
+
it "yields object and index" do
|
107
|
+
`ruby spec/cases/map_with_index.rb 2>&1`.should == 'a0b1'
|
108
|
+
end
|
109
|
+
|
110
|
+
it "does not crash with empty set" do
|
111
|
+
`ruby spec/cases/map_with_index_empty.rb 2>&1`.should == ''
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
describe :each do
|
116
|
+
it "returns original array, works like map" do
|
117
|
+
`ruby spec/cases/parallel_each.rb`.should == '-b--c--d--a-a b c d'
|
118
|
+
end
|
119
|
+
|
120
|
+
it "does not use marshal_dump" do
|
121
|
+
`ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'not dumpable'
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
describe :each_with_index do
|
126
|
+
it "yields object and index" do
|
127
|
+
`ruby spec/cases/each_with_index.rb 2>&1`.should == 'a0b1'
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
describe :in_groups_of do
|
132
|
+
it "works for empty" do
|
133
|
+
Parallel.send(:in_groups_of, [], 3).should == []
|
134
|
+
end
|
135
|
+
|
136
|
+
it "works for smaller then count" do
|
137
|
+
Parallel.send(:in_groups_of, [1,2], 3).should == [[1,2]]
|
138
|
+
end
|
139
|
+
|
140
|
+
it "works for count" do
|
141
|
+
Parallel.send(:in_groups_of, [1,2,3], 3).should == [[1,2,3]]
|
142
|
+
end
|
143
|
+
|
144
|
+
it "works for larger than count" do
|
145
|
+
Parallel.send(:in_groups_of, [1,2,3,4], 3).should == [[1,2,3],[4]]
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: lowang-parallel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 4
|
8
|
+
- 2
|
9
|
+
version: 0.4.2
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Michael Grosser
|
13
|
+
- Przemyslaw Wroblewski
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-05-10 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description:
|
23
|
+
email: przemyslaw.wroblewski@gmail.com
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files:
|
29
|
+
- README.markdown
|
30
|
+
files:
|
31
|
+
- .gitignore
|
32
|
+
- README.markdown
|
33
|
+
- Rakefile
|
34
|
+
- VERSION
|
35
|
+
- lib/parallel.rb
|
36
|
+
- lowang-parallel.gemspec
|
37
|
+
- parallel.gemspec
|
38
|
+
- spec/cases/each_with_index.rb
|
39
|
+
- spec/cases/map_with_index.rb
|
40
|
+
- spec/cases/map_with_index_empty.rb
|
41
|
+
- spec/cases/no_dump_with_each.rb
|
42
|
+
- spec/cases/parallel_each.rb
|
43
|
+
- spec/cases/parallel_high_fork_rate.rb
|
44
|
+
- spec/cases/parallel_influence_outside_data.rb
|
45
|
+
- spec/cases/parallel_map.rb
|
46
|
+
- spec/cases/parallel_map_range.rb
|
47
|
+
- spec/cases/parallel_map_sleeping.rb
|
48
|
+
- spec/cases/parallel_map_uneven.rb
|
49
|
+
- spec/cases/parallel_raise.rb
|
50
|
+
- spec/cases/parallel_sleeping_2.rb
|
51
|
+
- spec/cases/parallel_start_and_kill.rb
|
52
|
+
- spec/cases/parallel_with_detected_cpus.rb
|
53
|
+
- spec/cases/parallel_with_nil_uses_detected_cpus.rb
|
54
|
+
- spec/cases/parallel_with_set_processes.rb
|
55
|
+
- spec/parallel_spec.rb
|
56
|
+
- spec/spec_helper.rb
|
57
|
+
has_rdoc: true
|
58
|
+
homepage: http://github.com/lowang/lowang-parallel
|
59
|
+
licenses: []
|
60
|
+
|
61
|
+
post_install_message:
|
62
|
+
rdoc_options:
|
63
|
+
- --charset=UTF-8
|
64
|
+
require_paths:
|
65
|
+
- lib
|
66
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
segments:
|
71
|
+
- 0
|
72
|
+
version: "0"
|
73
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
segments:
|
78
|
+
- 0
|
79
|
+
version: "0"
|
80
|
+
requirements: []
|
81
|
+
|
82
|
+
rubyforge_project:
|
83
|
+
rubygems_version: 1.3.6
|
84
|
+
signing_key:
|
85
|
+
specification_version: 3
|
86
|
+
summary: Run any kind of code in parallel processes
|
87
|
+
test_files:
|
88
|
+
- spec/cases/each_with_index.rb
|
89
|
+
- spec/cases/parallel_with_nil_uses_detected_cpus.rb
|
90
|
+
- spec/cases/parallel_raise.rb
|
91
|
+
- spec/cases/parallel_map_uneven.rb
|
92
|
+
- spec/cases/parallel_high_fork_rate.rb
|
93
|
+
- spec/cases/map_with_index_empty.rb
|
94
|
+
- spec/cases/parallel_map_range.rb
|
95
|
+
- spec/cases/parallel_influence_outside_data.rb
|
96
|
+
- spec/cases/map_with_index.rb
|
97
|
+
- spec/cases/parallel_start_and_kill.rb
|
98
|
+
- spec/cases/parallel_with_detected_cpus.rb
|
99
|
+
- spec/cases/no_dump_with_each.rb
|
100
|
+
- spec/cases/parallel_map.rb
|
101
|
+
- spec/cases/parallel_each.rb
|
102
|
+
- spec/cases/parallel_sleeping_2.rb
|
103
|
+
- spec/cases/parallel_with_set_processes.rb
|
104
|
+
- spec/cases/parallel_map_sleeping.rb
|
105
|
+
- spec/parallel_spec.rb
|
106
|
+
- spec/spec_helper.rb
|