parallel 0.8.3 → 0.8.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/parallel.rb +1 -1
- data/lib/parallel/version.rb +1 -1
- metadata +2 -43
- data/.travis.yml +0 -5
- data/Gemfile +0 -6
- data/Gemfile.lock +0 -30
- data/MIT-LICENSE.txt +0 -20
- data/Rakefile +0 -7
- data/Readme.md +0 -119
- data/gem-public_cert.pem +0 -20
- data/parallel.gemspec +0 -17
- data/spec/cases/closes_processes_at_runtime.rb +0 -12
- data/spec/cases/count_open_pipes.rb +0 -6
- data/spec/cases/each.rb +0 -8
- data/spec/cases/each_with_index.rb +0 -5
- data/spec/cases/host_os_override_processor_count.rb +0 -4
- data/spec/cases/map_with_index.rb +0 -6
- data/spec/cases/map_with_index_empty.rb +0 -6
- data/spec/cases/map_with_killed_worker_before_read.rb +0 -9
- data/spec/cases/map_with_killed_worker_before_write.rb +0 -18
- data/spec/cases/map_with_nested_arrays_and_nil.rb +0 -7
- data/spec/cases/map_with_processes_and_break.rb +0 -10
- data/spec/cases/map_with_processes_and_exceptions.rb +0 -13
- data/spec/cases/map_with_threads_and_break.rb +0 -9
- data/spec/cases/map_with_threads_and_exceptions.rb +0 -12
- data/spec/cases/no_dump_with_each.rb +0 -21
- data/spec/cases/no_gc_with_each.rb +0 -5
- data/spec/cases/parallel_fast_exit.rb +0 -7
- data/spec/cases/parallel_high_fork_rate.rb +0 -6
- data/spec/cases/parallel_influence_outside_data.rb +0 -8
- data/spec/cases/parallel_map.rb +0 -6
- data/spec/cases/parallel_map_complex_objects.rb +0 -8
- data/spec/cases/parallel_map_range.rb +0 -6
- data/spec/cases/parallel_map_sleeping.rb +0 -5
- data/spec/cases/parallel_map_uneven.rb +0 -5
- data/spec/cases/parallel_raise.rb +0 -10
- data/spec/cases/parallel_raise_undumpable.rb +0 -17
- data/spec/cases/parallel_sleeping_2.rb +0 -5
- data/spec/cases/parallel_start_and_kill.rb +0 -12
- data/spec/cases/parallel_with_detected_cpus.rb +0 -6
- data/spec/cases/parallel_with_nil_uses_detected_cpus.rb +0 -6
- data/spec/cases/parallel_with_set_processes.rb +0 -6
- data/spec/parallel_spec.rb +0 -293
- data/spec/spec_helper.rb +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8dfa83f77faeb4b7a88c8fe48f4c4c761a698a79
|
4
|
+
data.tar.gz: b671b7078d94652f32c4d3926389ae49a8757ec2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2cf2bef92031948c03f45f6fae48637f16042ed84eaf97aa88894ac35e1db35fcd8e4e12b43819efdf8a9fccfc3af4aa23329808ac174935c8184a349ee1660
|
7
|
+
data.tar.gz: 5baf3578ead853ac7643d176ffcfd38e9f73a3c96c622e2af14517f0dbad756e85ae2b62b45582b9be32fae9d1c4e3c9309dfb2bf93fc546a15178baedf6adae
|
data/lib/parallel.rb
CHANGED
@@ -137,7 +137,7 @@ module Parallel
|
|
137
137
|
def physical_processor_count
|
138
138
|
@physical_processor_count ||= begin
|
139
139
|
ppc = case RbConfig::CONFIG['host_os']
|
140
|
-
when /darwin1
|
140
|
+
when /darwin1/
|
141
141
|
`sysctl -n hw.physicalcpu`.to_i
|
142
142
|
when /linux/
|
143
143
|
cores_per_physical = `grep cores /proc/cpuinfo`[/\d+/].to_i
|
data/lib/parallel/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-09-
|
11
|
+
date: 2013-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: michael@grosser.it
|
@@ -16,49 +16,8 @@ executables: []
|
|
16
16
|
extensions: []
|
17
17
|
extra_rdoc_files: []
|
18
18
|
files:
|
19
|
-
- .travis.yml
|
20
|
-
- Gemfile
|
21
|
-
- Gemfile.lock
|
22
|
-
- MIT-LICENSE.txt
|
23
|
-
- Rakefile
|
24
|
-
- Readme.md
|
25
|
-
- gem-public_cert.pem
|
26
19
|
- lib/parallel.rb
|
27
20
|
- lib/parallel/version.rb
|
28
|
-
- parallel.gemspec
|
29
|
-
- spec/cases/closes_processes_at_runtime.rb
|
30
|
-
- spec/cases/count_open_pipes.rb
|
31
|
-
- spec/cases/each.rb
|
32
|
-
- spec/cases/each_with_index.rb
|
33
|
-
- spec/cases/host_os_override_processor_count.rb
|
34
|
-
- spec/cases/map_with_index.rb
|
35
|
-
- spec/cases/map_with_index_empty.rb
|
36
|
-
- spec/cases/map_with_killed_worker_before_read.rb
|
37
|
-
- spec/cases/map_with_killed_worker_before_write.rb
|
38
|
-
- spec/cases/map_with_nested_arrays_and_nil.rb
|
39
|
-
- spec/cases/map_with_processes_and_break.rb
|
40
|
-
- spec/cases/map_with_processes_and_exceptions.rb
|
41
|
-
- spec/cases/map_with_threads_and_break.rb
|
42
|
-
- spec/cases/map_with_threads_and_exceptions.rb
|
43
|
-
- spec/cases/no_dump_with_each.rb
|
44
|
-
- spec/cases/no_gc_with_each.rb
|
45
|
-
- spec/cases/parallel_fast_exit.rb
|
46
|
-
- spec/cases/parallel_high_fork_rate.rb
|
47
|
-
- spec/cases/parallel_influence_outside_data.rb
|
48
|
-
- spec/cases/parallel_map.rb
|
49
|
-
- spec/cases/parallel_map_complex_objects.rb
|
50
|
-
- spec/cases/parallel_map_range.rb
|
51
|
-
- spec/cases/parallel_map_sleeping.rb
|
52
|
-
- spec/cases/parallel_map_uneven.rb
|
53
|
-
- spec/cases/parallel_raise.rb
|
54
|
-
- spec/cases/parallel_raise_undumpable.rb
|
55
|
-
- spec/cases/parallel_sleeping_2.rb
|
56
|
-
- spec/cases/parallel_start_and_kill.rb
|
57
|
-
- spec/cases/parallel_with_detected_cpus.rb
|
58
|
-
- spec/cases/parallel_with_nil_uses_detected_cpus.rb
|
59
|
-
- spec/cases/parallel_with_set_processes.rb
|
60
|
-
- spec/parallel_spec.rb
|
61
|
-
- spec/spec_helper.rb
|
62
21
|
homepage: https://github.com/grosser/parallel
|
63
22
|
licenses:
|
64
23
|
- MIT
|
data/.travis.yml
DELETED
data/Gemfile
DELETED
data/Gemfile.lock
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
parallel (0.8.3)
|
5
|
-
|
6
|
-
GEM
|
7
|
-
remote: https://rubygems.org/
|
8
|
-
specs:
|
9
|
-
bump (0.3.5)
|
10
|
-
diff-lcs (1.1.3)
|
11
|
-
rake (10.0.2)
|
12
|
-
rspec (2.12.0)
|
13
|
-
rspec-core (~> 2.12.0)
|
14
|
-
rspec-expectations (~> 2.12.0)
|
15
|
-
rspec-mocks (~> 2.12.0)
|
16
|
-
rspec-core (2.12.0)
|
17
|
-
rspec-expectations (2.12.0)
|
18
|
-
diff-lcs (~> 1.1.3)
|
19
|
-
rspec-mocks (2.12.0)
|
20
|
-
|
21
|
-
PLATFORMS
|
22
|
-
java
|
23
|
-
ruby
|
24
|
-
x86-mingw32
|
25
|
-
|
26
|
-
DEPENDENCIES
|
27
|
-
bump
|
28
|
-
parallel!
|
29
|
-
rake
|
30
|
-
rspec (~> 2)
|
data/MIT-LICENSE.txt
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
Copyright (C) 2013 Michael Grosser <michael@grosser.it>
|
2
|
-
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
-
a copy of this software and associated documentation files (the
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
9
|
-
the following conditions:
|
10
|
-
|
11
|
-
The above copyright notice and this permission notice shall be
|
12
|
-
included in all copies or substantial portions of the Software.
|
13
|
-
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
DELETED
data/Readme.md
DELETED
@@ -1,119 +0,0 @@
|
|
1
|
-
Run any code in parallel Processes(> use all CPUs) or Threads(> speedup blocking operations).<br/>
|
2
|
-
Best suited for map-reduce or e.g. parallel downloads/uploads.
|
3
|
-
|
4
|
-
Install
|
5
|
-
=======
|
6
|
-
|
7
|
-
```Bash
|
8
|
-
gem install parallel
|
9
|
-
```
|
10
|
-
|
11
|
-
Usage
|
12
|
-
=====
|
13
|
-
|
14
|
-
```Ruby
|
15
|
-
# 2 CPUs -> work in 2 processes (a,b + c)
|
16
|
-
results = Parallel.map(['a','b','c']) do |one_letter|
|
17
|
-
expensive_calculation(one_letter)
|
18
|
-
end
|
19
|
-
|
20
|
-
# 3 Processes -> finished after 1 run
|
21
|
-
results = Parallel.map(['a','b','c'], :in_processes=>3){|one_letter| ... }
|
22
|
-
|
23
|
-
# 3 Threads -> finished after 1 run
|
24
|
-
results = Parallel.map(['a','b','c'], :in_threads=>3){|one_letter| ... }
|
25
|
-
```
|
26
|
-
|
27
|
-
Same can be done with `each`
|
28
|
-
```Ruby
|
29
|
-
Parallel.each(['a','b','c']){|one_letter| ... }
|
30
|
-
```
|
31
|
-
or `each_with_index` or `map_with_index`
|
32
|
-
|
33
|
-
Processes/Threads are workers, they grab the next piece of work when they finish.
|
34
|
-
|
35
|
-
### Processes
|
36
|
-
- Speedup through multiple CPUs
|
37
|
-
- Speedup for blocking operations
|
38
|
-
- Protects global data
|
39
|
-
- Extra memory used ( very low on [REE](http://www.rubyenterpriseedition.com/faq.html) through `copy_on_write_friendly` )
|
40
|
-
- Child processes are killed when your main process is killed through Ctrl+c or kill -2
|
41
|
-
|
42
|
-
### Threads
|
43
|
-
- Speedup for blocking operations
|
44
|
-
- Global data can be modified
|
45
|
-
- No extra memory used
|
46
|
-
|
47
|
-
### ActiveRecord
|
48
|
-
|
49
|
-
Try either of those to get working parallel AR
|
50
|
-
|
51
|
-
```Ruby
|
52
|
-
Parallel.each(User.all, :in_threads => 8) do |user|
|
53
|
-
ActiveRecord::Base.connection_pool.with_connection do
|
54
|
-
user.update_attribute(:some_attribute, some_value)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
Parallel.each(User.all, :in_processes => 8) do |user|
|
59
|
-
ActiveRecord::Base.connection.reconnect!
|
60
|
-
user.update_attribute(:some_attribute, some_value)
|
61
|
-
end
|
62
|
-
```
|
63
|
-
|
64
|
-
### Break
|
65
|
-
|
66
|
-
```Ruby
|
67
|
-
Parallel.map(User.all) do |user|
|
68
|
-
raise Parallel::Break # -> stop all execution
|
69
|
-
end
|
70
|
-
```
|
71
|
-
|
72
|
-
### Progress / ETA
|
73
|
-
|
74
|
-
```Bash
|
75
|
-
gem install ruby-progressbar
|
76
|
-
```
|
77
|
-
|
78
|
-
```Ruby
|
79
|
-
require 'ruby-progressbar'
|
80
|
-
progress = ProgressBar.create(:title => "The Progress", :total => 100)
|
81
|
-
Parallel.map(1..100, :finish => lambda { |item, i| progress.increment }) { sleep 1 }
|
82
|
-
```
|
83
|
-
|
84
|
-
Tips
|
85
|
-
====
|
86
|
-
- [Benchmark/Test] Disable threading/forking with `:in_threads => 0` or `:in_processes => 0`, great to test performance or to debug parallel issues
|
87
|
-
|
88
|
-
TODO
|
89
|
-
====
|
90
|
-
- JRuby / Windows support <-> possible ?
|
91
|
-
|
92
|
-
Authors
|
93
|
-
=======
|
94
|
-
|
95
|
-
### [Contributors](https://github.com/grosser/parallel/contributors)
|
96
|
-
- [Przemyslaw Wroblewski](http://github.com/lowang)
|
97
|
-
- [TJ Holowaychuk](http://vision-media.ca/)
|
98
|
-
- [Masatomo Nakano](http://twitter.com/masatomo2)
|
99
|
-
- [Fred Wu](http://fredwu.me)
|
100
|
-
- [mikezter](http://github.com/mikezter)
|
101
|
-
- [Jeremy Durham](http://www.jeremydurham.com)
|
102
|
-
- [Nick Gauthier](http://www.ngauthier.com)
|
103
|
-
- [Andrew Bowerman](http://andrewbowerman.com)
|
104
|
-
- [Byron Bowerman](http://me.bm5k.com/)
|
105
|
-
- [Mikko Kokkonen](https://github.com/mikian)
|
106
|
-
- [brian p o'rourke](https://github.com/bpo)
|
107
|
-
- [Norio Sato]
|
108
|
-
- [Neal Stewart](https://github.com/n-time)
|
109
|
-
- [Jurriaan Pruis](http://github.com/jurriaan)
|
110
|
-
- [Rob Worley](http://github.com/robworley)
|
111
|
-
- [Tasveer Singh](https://github.com/tazsingh)
|
112
|
-
- [Joachim](https://github.com/jmozmoz)
|
113
|
-
- [yaoguai](https://github.com/yaoguai)
|
114
|
-
- [Bartosz Dziewoński](https://github.com/MatmaRex)
|
115
|
-
|
116
|
-
[Michael Grosser](http://grosser.it)<br/>
|
117
|
-
michael@grosser.it<br/>
|
118
|
-
License: MIT<br/>
|
119
|
-
[![Build Status](https://travis-ci.org/grosser/parallel.png)](https://travis-ci.org/grosser/parallel)
|
data/gem-public_cert.pem
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
-----BEGIN CERTIFICATE-----
|
2
|
-
MIIDMjCCAhqgAwIBAgIBADANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
|
3
|
-
YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
|
4
|
-
MB4XDTEzMDIwMzE4MTMxMVoXDTE0MDIwMzE4MTMxMVowPzEQMA4GA1UEAwwHbWlj
|
5
|
-
aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
|
6
|
-
dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
|
7
|
-
MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
|
8
|
-
cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
|
9
|
-
6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
|
10
|
-
h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
|
11
|
-
FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
|
12
|
-
/88CAwEAAaM5MDcwCQYDVR0TBAIwADAdBgNVHQ4EFgQUsiNnXHtKeMYYcr4yJVmQ
|
13
|
-
WONL+IwwCwYDVR0PBAQDAgSwMA0GCSqGSIb3DQEBBQUAA4IBAQAlyN7kKo/NQCQ0
|
14
|
-
AOzZLZ3WAePvStkCFIJ53tsv5Kyo4pMAllv+BgPzzBt7qi605mFSL6zBd9uLou+W
|
15
|
-
Co3s48p1dy7CjjAfVQdmVNHF3MwXtfC2OEyvSQPi4xKR8iba8wa3xp9LVo1PuLpw
|
16
|
-
/6DsrChWw74HfsJN6qJOK684hJeT8lBYAUfiC3wD0owoPSg+XtyAAddisR+KV5Y1
|
17
|
-
NmVHuLtQcNTZy+gRht3ahJRMuC6QyLmkTsf+6MaenwAMkAgHdswGsJztOnNnBa3F
|
18
|
-
y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
|
19
|
-
ycwMXfl0
|
20
|
-
-----END CERTIFICATE-----
|
data/parallel.gemspec
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
|
2
|
-
name = "parallel"
|
3
|
-
require "#{name}/version"
|
4
|
-
|
5
|
-
Gem::Specification.new name, Parallel::VERSION do |s|
|
6
|
-
s.summary = "Run any kind of code in parallel processes"
|
7
|
-
s.authors = ["Michael Grosser"]
|
8
|
-
s.email = "michael@grosser.it"
|
9
|
-
s.homepage = "https://github.com/grosser/#{name}"
|
10
|
-
s.files = `git ls-files`.split("\n")
|
11
|
-
s.license = "MIT"
|
12
|
-
key = File.expand_path("~/.ssh/gem-private_key.pem")
|
13
|
-
if File.exist?(key)
|
14
|
-
s.signing_key = key
|
15
|
-
s.cert_chain = ["gem-public_cert.pem"]
|
16
|
-
end
|
17
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
cmd = "ps uaxw|grep ruby|wc -l"
|
3
|
-
|
4
|
-
processes_before = `#{cmd}`.to_i
|
5
|
-
Parallel.each((0..10).to_a, :in_processes => 5) { |a| a*2 }
|
6
|
-
processes_after = `#{cmd}`.to_i
|
7
|
-
|
8
|
-
if processes_before == processes_after
|
9
|
-
print 'OK'
|
10
|
-
else
|
11
|
-
print "FAIL: before:#{processes_before} -- after:#{processes_after}"
|
12
|
-
end
|
data/spec/cases/each.rb
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
Parallel::Worker.class_eval do
|
4
|
-
alias_method :work_without_kill, :work
|
5
|
-
def work(*args)
|
6
|
-
Process.kill("SIGKILL", pid)
|
7
|
-
sleep 0.5
|
8
|
-
work_without_kill(*args)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
begin
|
13
|
-
Parallel.map([1,2,3]) do |x, i|
|
14
|
-
Process.kill("SIGKILL", Process.pid)
|
15
|
-
end
|
16
|
-
rescue Parallel::DeadWorker
|
17
|
-
puts "DEAD"
|
18
|
-
end
|
@@ -1,10 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
result = Parallel.map(1..100, :in_processes => 4) do |x|
|
4
|
-
sleep 0.1 # so all processes get started
|
5
|
-
print x
|
6
|
-
raise Parallel::Break if x == 1
|
7
|
-
sleep 0.1 # so no now work gets queued before Parallel::Break is raised
|
8
|
-
x
|
9
|
-
end
|
10
|
-
print " Parallel::Break raised - result #{result.inspect}"
|
@@ -1,13 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
begin
|
4
|
-
Parallel.map(1..100, :in_processes => 4) do |x|
|
5
|
-
sleep 0.1 # so all processes get started
|
6
|
-
print x
|
7
|
-
raise 'foo' if x == 1
|
8
|
-
sleep 0.1 # so no now work gets queued before exception is raised
|
9
|
-
x
|
10
|
-
end
|
11
|
-
rescue
|
12
|
-
print ' raised'
|
13
|
-
end
|
@@ -1,9 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
result = Parallel.map(1..100, :in_threads => 4) do |x|
|
4
|
-
sleep 0.1 # so all threads get started
|
5
|
-
print x
|
6
|
-
raise Parallel::Break if x == 1
|
7
|
-
sleep 0.1 # so no now work gets queued before Parallel::Break is raised
|
8
|
-
end
|
9
|
-
print " Parallel::Break raised - result #{result.inspect}"
|
@@ -1,12 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
begin
|
4
|
-
Parallel.map(1..100, :in_threads => 4) do |x|
|
5
|
-
sleep 0.1 # so all processes get started
|
6
|
-
print x
|
7
|
-
raise 'foo' if x == 1
|
8
|
-
sleep 0.1 # so no now work gets queued before exception is raised
|
9
|
-
end
|
10
|
-
rescue
|
11
|
-
print ' raised'
|
12
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
class NotDumpable
|
4
|
-
def marshal_dump
|
5
|
-
raise "NOOOO"
|
6
|
-
end
|
7
|
-
|
8
|
-
def to_s
|
9
|
-
'not dumpable'
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
Parallel.each([1]) do
|
14
|
-
print 'no dump for result'
|
15
|
-
NotDumpable.new
|
16
|
-
end
|
17
|
-
|
18
|
-
Parallel.each([NotDumpable.new]) do
|
19
|
-
print 'no dump for each'
|
20
|
-
1
|
21
|
-
end
|
data/spec/cases/parallel_map.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
require 'stringio'
|
3
|
-
|
4
|
-
class MyException < Exception
|
5
|
-
def initialize(object)
|
6
|
-
@object = object
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
begin
|
11
|
-
Parallel.in_processes(2) do
|
12
|
-
raise MyException.new(StringIO.new)
|
13
|
-
end
|
14
|
-
puts "FAIL"
|
15
|
-
rescue RuntimeError
|
16
|
-
puts $!.message
|
17
|
-
end
|
@@ -1,12 +0,0 @@
|
|
1
|
-
require File.expand_path('spec/spec_helper')
|
2
|
-
|
3
|
-
method = case ARGV[0]
|
4
|
-
when "PROCESS" then :in_processes
|
5
|
-
when "THREAD" then :in_threads
|
6
|
-
else raise "Learn to use this!"
|
7
|
-
end
|
8
|
-
|
9
|
-
Parallel.send(method, 2) do
|
10
|
-
sleep 10
|
11
|
-
puts "I should have been killed earlier..."
|
12
|
-
end
|
data/spec/parallel_spec.rb
DELETED
@@ -1,293 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Parallel do
|
4
|
-
def time_taken
|
5
|
-
t = Time.now.to_f
|
6
|
-
yield
|
7
|
-
Time.now.to_f - t
|
8
|
-
end
|
9
|
-
|
10
|
-
def kill_process_with_name(file)
|
11
|
-
running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) }
|
12
|
-
pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1
|
13
|
-
parent_pid = running_processes.detect { |p| p.include?(file) and not p.include?("sh") }[pid_index]
|
14
|
-
`kill -2 #{parent_pid}`
|
15
|
-
end
|
16
|
-
|
17
|
-
describe ".processor_count" do
|
18
|
-
it "returns a number" do
|
19
|
-
(1..999).should include(Parallel.processor_count)
|
20
|
-
end
|
21
|
-
|
22
|
-
if RUBY_PLATFORM =~ /darwin10/
|
23
|
-
it 'works if hwprefs in not available' do
|
24
|
-
Parallel.should_receive(:hwprefs_available?).and_return false
|
25
|
-
(1..999).should include(Parallel.processor_count)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
it "defaults to 1 if we don't know better" do
|
30
|
-
`ruby spec/cases/host_os_override_processor_count.rb 2>/dev/null`.to_i.should == 1
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
describe ".physical_processor_count" do
|
35
|
-
it "returns a number" do
|
36
|
-
(1..999).should include(Parallel.physical_processor_count)
|
37
|
-
end
|
38
|
-
|
39
|
-
it "is even factor of logical cpus" do
|
40
|
-
pending if ENV["TRAVIS"]
|
41
|
-
(Parallel.processor_count % Parallel.physical_processor_count).should == 0
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
describe ".in_processes" do
|
46
|
-
def cpus
|
47
|
-
Parallel.processor_count
|
48
|
-
end
|
49
|
-
|
50
|
-
it "executes with detected cpus" do
|
51
|
-
`ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * cpus
|
52
|
-
end
|
53
|
-
|
54
|
-
it "executes with detected cpus when nil was given" do
|
55
|
-
`ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * cpus
|
56
|
-
end
|
57
|
-
|
58
|
-
it "set amount of parallel processes" do
|
59
|
-
`ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
|
60
|
-
end
|
61
|
-
|
62
|
-
it "does not influence outside data" do
|
63
|
-
`ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
|
64
|
-
end
|
65
|
-
|
66
|
-
it "kills the processes when the main process gets killed through ctrl+c" do
|
67
|
-
time_taken{
|
68
|
-
lambda{
|
69
|
-
t = Thread.new { `ruby spec/cases/parallel_start_and_kill.rb PROCESS` }
|
70
|
-
sleep 1
|
71
|
-
kill_process_with_name("spec/cases/parallel_start_and_kill.rb") #simulates Ctrl+c
|
72
|
-
sleep 1
|
73
|
-
puts t.value
|
74
|
-
}.should_not change{`ps`.split("\n").size}
|
75
|
-
}.should <= 3
|
76
|
-
end
|
77
|
-
|
78
|
-
it "kills the threads when the main process gets killed through ctrl+c" do
|
79
|
-
time_taken{
|
80
|
-
lambda{
|
81
|
-
Thread.new { `ruby spec/cases/parallel_start_and_kill.rb THREAD` }
|
82
|
-
sleep 1
|
83
|
-
kill_process_with_name("spec/cases/parallel_start_and_kill.rb") #simulates Ctrl+c
|
84
|
-
sleep 1
|
85
|
-
}.should_not change{`ps`.split("\n").size}
|
86
|
-
}.should <= 3
|
87
|
-
end
|
88
|
-
|
89
|
-
it "does not kill anything on ctrl+c when everything has finished" do
|
90
|
-
time_taken do
|
91
|
-
t = Thread.new { `ruby spec/cases/parallel_fast_exit.rb 2>&1` }
|
92
|
-
sleep 2
|
93
|
-
kill_process_with_name("spec/cases/parallel_fast_exit.rb") #simulates Ctrl+c
|
94
|
-
sleep 1
|
95
|
-
result = t.value
|
96
|
-
result.scan(/I finished/).size.should == 3
|
97
|
-
result.should_not include("Parallel execution interrupted")
|
98
|
-
end.should <= 4
|
99
|
-
end
|
100
|
-
|
101
|
-
it "saves time" do
|
102
|
-
time_taken{
|
103
|
-
`ruby spec/cases/parallel_sleeping_2.rb`
|
104
|
-
}.should < 3.5
|
105
|
-
end
|
106
|
-
|
107
|
-
it "raises when one of the processes raises" do
|
108
|
-
`ruby spec/cases/parallel_raise.rb`.strip.should == 'TEST'
|
109
|
-
end
|
110
|
-
|
111
|
-
it "can raise an undumpable exception" do
|
112
|
-
`ruby spec/cases/parallel_raise_undumpable.rb`.strip.should include('Undumpable Exception')
|
113
|
-
end
|
114
|
-
|
115
|
-
it 'can handle to high fork rate' do
|
116
|
-
`ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
|
117
|
-
end
|
118
|
-
|
119
|
-
it 'does not leave processes behind while running' do
|
120
|
-
`ruby spec/cases/closes_processes_at_runtime.rb`.should == 'OK'
|
121
|
-
end
|
122
|
-
|
123
|
-
it "does not open unnecessary pipes" do
|
124
|
-
open_pipes = `lsof | grep pipe | wc -l`.to_i
|
125
|
-
max_pipes = `ruby spec/cases/count_open_pipes.rb`.to_i
|
126
|
-
(max_pipes - open_pipes).should < 400
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
describe ".in_threads" do
|
131
|
-
it "saves time" do
|
132
|
-
time_taken{
|
133
|
-
Parallel.in_threads(3){ sleep 2 }
|
134
|
-
}.should < 3
|
135
|
-
end
|
136
|
-
|
137
|
-
it "does not create new processes" do
|
138
|
-
lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
|
139
|
-
end
|
140
|
-
|
141
|
-
it "returns results as array" do
|
142
|
-
Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
|
143
|
-
end
|
144
|
-
|
145
|
-
it "raises when a thread raises" do
|
146
|
-
lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
|
147
|
-
end
|
148
|
-
end
|
149
|
-
|
150
|
-
describe ".map" do
|
151
|
-
it "saves time" do
|
152
|
-
time_taken{
|
153
|
-
`ruby spec/cases/parallel_map_sleeping.rb`
|
154
|
-
}.should <= 3.5
|
155
|
-
end
|
156
|
-
|
157
|
-
it "executes with given parameters" do
|
158
|
-
`ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
|
159
|
-
end
|
160
|
-
|
161
|
-
it "can dump/load complex objects" do
|
162
|
-
`ruby spec/cases/parallel_map_complex_objects.rb`.should == "YES"
|
163
|
-
end
|
164
|
-
|
165
|
-
it "starts new process imediatly when old exists" do
|
166
|
-
time_taken{
|
167
|
-
`ruby spec/cases/parallel_map_uneven.rb`
|
168
|
-
}.should <= 3.5
|
169
|
-
end
|
170
|
-
|
171
|
-
it "does not flatten results" do
|
172
|
-
Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
|
173
|
-
end
|
174
|
-
|
175
|
-
it "can run in threads" do
|
176
|
-
Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
177
|
-
end
|
178
|
-
|
179
|
-
it 'supports all Enumerable-s' do
|
180
|
-
`ruby spec/cases/parallel_map_range.rb`.should == '[1, 2, 3, 4, 5]'
|
181
|
-
end
|
182
|
-
|
183
|
-
it 'handles nested arrays and nil correctly' do
|
184
|
-
`ruby spec/cases/map_with_nested_arrays_and_nil.rb`.should == '[nil, [2, 2], [[3], [3]]]'
|
185
|
-
end
|
186
|
-
|
187
|
-
it 'stops all workers when one fails in process' do
|
188
|
-
`ruby spec/cases/map_with_processes_and_exceptions.rb 2>&1`.should =~ /^\d{4} raised$/
|
189
|
-
end
|
190
|
-
|
191
|
-
it 'stops all workers when one fails in thread' do
|
192
|
-
`ruby spec/cases/map_with_threads_and_exceptions.rb 2>&1`.should =~ /^\d{0,4} raised$/
|
193
|
-
end
|
194
|
-
|
195
|
-
it 'stops all workers when one raises Break in process' do
|
196
|
-
`ruby spec/cases/map_with_processes_and_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result nil$/
|
197
|
-
end
|
198
|
-
|
199
|
-
it 'stops all workers when one raises Break in thread' do
|
200
|
-
`ruby spec/cases/map_with_threads_and_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result nil$/
|
201
|
-
end
|
202
|
-
|
203
|
-
it "can run with 0 threads" do
|
204
|
-
Thread.should_not_receive(:exclusive)
|
205
|
-
Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
206
|
-
end
|
207
|
-
|
208
|
-
it "can run with 0 processes" do
|
209
|
-
Process.should_not_receive(:fork)
|
210
|
-
Parallel.map([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
211
|
-
end
|
212
|
-
|
213
|
-
it "notifies when an item of work is dispatched to a worker process" do
|
214
|
-
monitor = double('monitor', :call => nil)
|
215
|
-
monitor.should_receive(:call).once.with(:first, 0)
|
216
|
-
monitor.should_receive(:call).once.with(:second, 1)
|
217
|
-
monitor.should_receive(:call).once.with(:third, 2)
|
218
|
-
Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 3) {}
|
219
|
-
end
|
220
|
-
|
221
|
-
it "notifies when an item of work is completed by a worker process" do
|
222
|
-
monitor = double('monitor', :call => nil)
|
223
|
-
monitor.should_receive(:call).once.with(:first, 0)
|
224
|
-
monitor.should_receive(:call).once.with(:second, 1)
|
225
|
-
monitor.should_receive(:call).once.with(:third, 2)
|
226
|
-
Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 3) {}
|
227
|
-
end
|
228
|
-
|
229
|
-
it "notifies when an item of work is dispatched to a threaded worker" do
|
230
|
-
monitor = double('monitor', :call => nil)
|
231
|
-
monitor.should_receive(:call).once.with(:first, 0)
|
232
|
-
monitor.should_receive(:call).once.with(:second, 1)
|
233
|
-
monitor.should_receive(:call).once.with(:third, 2)
|
234
|
-
Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 3) {}
|
235
|
-
end
|
236
|
-
|
237
|
-
it "notifies when an item of work is completed by a threaded worker" do
|
238
|
-
monitor = double('monitor', :call => nil)
|
239
|
-
monitor.should_receive(:call).once.with(:first, 0)
|
240
|
-
monitor.should_receive(:call).once.with(:second, 1)
|
241
|
-
monitor.should_receive(:call).once.with(:third, 2)
|
242
|
-
Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 3) {}
|
243
|
-
end
|
244
|
-
|
245
|
-
it "spits out a useful error when a worker dies before read" do
|
246
|
-
`ruby spec/cases/map_with_killed_worker_before_read.rb 2>&1`.should include "DEAD"
|
247
|
-
end
|
248
|
-
|
249
|
-
it "spits out a useful error when a worker dies before write" do
|
250
|
-
`ruby spec/cases/map_with_killed_worker_before_write.rb 2>&1`.should include "DEAD"
|
251
|
-
end
|
252
|
-
end
|
253
|
-
|
254
|
-
describe ".map_with_index" do
|
255
|
-
it "yields object and index" do
|
256
|
-
`ruby spec/cases/map_with_index.rb 2>&1`.should == 'a0b1'
|
257
|
-
end
|
258
|
-
|
259
|
-
it "does not crash with empty set" do
|
260
|
-
`ruby spec/cases/map_with_index_empty.rb 2>&1`.should == ''
|
261
|
-
end
|
262
|
-
|
263
|
-
it "can run with 0 threads" do
|
264
|
-
Thread.should_not_receive(:exclusive)
|
265
|
-
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
266
|
-
end
|
267
|
-
|
268
|
-
it "can run with 0 processes" do
|
269
|
-
Process.should_not_receive(:fork)
|
270
|
-
Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
271
|
-
end
|
272
|
-
end
|
273
|
-
|
274
|
-
describe ".each" do
|
275
|
-
it "returns original array, works like map" do
|
276
|
-
`ruby spec/cases/each.rb`.should == 'a b c d'
|
277
|
-
end
|
278
|
-
|
279
|
-
it "does not use marshal_dump" do
|
280
|
-
`ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'no dump for resultno dump for each'
|
281
|
-
end
|
282
|
-
|
283
|
-
it "does not slow down with lots of GC work in threads" do
|
284
|
-
Benchmark.realtime { `ruby spec/cases/no_gc_with_each.rb 2>&1` }.should <= 10
|
285
|
-
end
|
286
|
-
end
|
287
|
-
|
288
|
-
describe ".each_with_index" do
|
289
|
-
it "yields object and index" do
|
290
|
-
["a0b1", "b1a0"].should include `ruby spec/cases/each_with_index.rb 2>&1`
|
291
|
-
end
|
292
|
-
end
|
293
|
-
end
|
data/spec/spec_helper.rb
DELETED