parallel 0.5.19 → 0.5.20
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/Gemfile.lock +11 -9
- data/Rakefile +2 -18
- data/Readme.md +26 -1
- data/lib/parallel.rb +14 -0
- data/lib/parallel/version.rb +1 -1
- data/spec/parallel_spec.rb +32 -0
- metadata +4 -4
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,26 +1,28 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
parallel (0.5.
|
4
|
+
parallel (0.5.20)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: http://rubygems.org/
|
8
8
|
specs:
|
9
|
+
bump (0.3.5)
|
9
10
|
diff-lcs (1.1.3)
|
10
|
-
rake (0.
|
11
|
-
rspec (2.
|
12
|
-
rspec-core (~> 2.
|
13
|
-
rspec-expectations (~> 2.
|
14
|
-
rspec-mocks (~> 2.
|
15
|
-
rspec-core (2.
|
16
|
-
rspec-expectations (2.
|
11
|
+
rake (10.0.2)
|
12
|
+
rspec (2.12.0)
|
13
|
+
rspec-core (~> 2.12.0)
|
14
|
+
rspec-expectations (~> 2.12.0)
|
15
|
+
rspec-mocks (~> 2.12.0)
|
16
|
+
rspec-core (2.12.0)
|
17
|
+
rspec-expectations (2.12.0)
|
17
18
|
diff-lcs (~> 1.1.3)
|
18
|
-
rspec-mocks (2.
|
19
|
+
rspec-mocks (2.12.0)
|
19
20
|
|
20
21
|
PLATFORMS
|
21
22
|
ruby
|
22
23
|
|
23
24
|
DEPENDENCIES
|
25
|
+
bump
|
24
26
|
parallel!
|
25
27
|
rake
|
26
28
|
rspec (~> 2)
|
data/Rakefile
CHANGED
@@ -1,23 +1,7 @@
|
|
1
|
+
require 'bundler/setup'
|
1
2
|
require 'bundler/gem_tasks'
|
3
|
+
require 'bump/tasks'
|
2
4
|
|
3
5
|
task :default do
|
4
6
|
sh "rspec spec/"
|
5
7
|
end
|
6
|
-
|
7
|
-
# extracted from https://github.com/grosser/project_template
|
8
|
-
rule /^version:bump:.*/ do |t|
|
9
|
-
sh "git status | grep 'nothing to commit'" # ensure we are not dirty
|
10
|
-
index = ['major', 'minor','patch'].index(t.name.split(':').last)
|
11
|
-
file = 'lib/parallel/version.rb'
|
12
|
-
|
13
|
-
version_file = File.read(file)
|
14
|
-
old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
|
15
|
-
version_parts[index] = version_parts[index].to_i + 1
|
16
|
-
version_parts[2] = 0 if index < 2 # remove patch for minor
|
17
|
-
version_parts[1] = 0 if index < 1 # remove minor for major
|
18
|
-
new_version = version_parts * '.'
|
19
|
-
File.open(file,'w'){|f| f.write(version_file.sub(old_version, new_version)) }
|
20
|
-
|
21
|
-
sh "bundle && git add #{file} Gemfile.lock && git commit -m 'bump version to #{new_version}'"
|
22
|
-
end
|
23
|
-
|
data/Readme.md
CHANGED
@@ -34,12 +34,36 @@ or `each_with_index` or `map_with_index`
|
|
34
34
|
- Global data can be modified
|
35
35
|
- No extra memory used
|
36
36
|
|
37
|
+
### ActiveRecord
|
38
|
+
|
39
|
+
Try either of those to get working parallel AR
|
40
|
+
|
41
|
+
```Ruby
|
42
|
+
Parallel.each(User.all, :in_threads => 8) do |user|
|
43
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
44
|
+
user.update_attribute(:some_attribute, some_value)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
Parallel.each(User.all, :in_processes => 8) do |user|
|
49
|
+
ActiveRecord::Base.connection.reconnect!
|
50
|
+
user.update_attribute(:some_attribute, some_value)
|
51
|
+
end
|
52
|
+
```
|
37
53
|
|
38
54
|
Processes/Threads are workers, they grab the next piece of work when they finish
|
39
55
|
|
56
|
+
### Progress / ETA
|
57
|
+
|
58
|
+
```Ruby
|
59
|
+
require 'progressbar'
|
60
|
+
progress = ProgressBar.new("test", 100)
|
61
|
+
Parallel.map(1..100, :finish => lambda { |i, item| progress.inc }) { sleep 1 }
|
62
|
+
progress.finish
|
63
|
+
```
|
64
|
+
|
40
65
|
Tips
|
41
66
|
====
|
42
|
-
- [ActiveRecord] `ActiveRecord::Base.connection.reconnect!` inside the parallel block prevents errors
|
43
67
|
- [Benchmark/Test] Disable threading/forking with `:in_threads => 0` or `:in_processes => 0`, great to test performance or to debug parallel issues
|
44
68
|
|
45
69
|
TODO
|
@@ -64,6 +88,7 @@ Authors
|
|
64
88
|
- [Norio Sato]
|
65
89
|
- [Neal Stewart](https://github.com/n-time)
|
66
90
|
- [Jurriaan Pruis](http://github.com/jurriaan)
|
91
|
+
- [Rob Worley](http://github.com/robworley)
|
67
92
|
|
68
93
|
[Michael Grosser](http://grosser.it)<br/>
|
69
94
|
michael@grosser.it<br/>
|
data/lib/parallel.rb
CHANGED
@@ -117,6 +117,8 @@ module Parallel
|
|
117
117
|
results = []
|
118
118
|
current = -1
|
119
119
|
exception = nil
|
120
|
+
on_start = options[:start]
|
121
|
+
on_finish = options[:finish]
|
120
122
|
|
121
123
|
in_threads(options[:count]) do
|
122
124
|
# as long as there are more items, work on one of them
|
@@ -126,11 +128,16 @@ module Parallel
|
|
126
128
|
index = Thread.exclusive{ current+=1 }
|
127
129
|
break if index >= items.size
|
128
130
|
|
131
|
+
item = items[index]
|
132
|
+
on_start.call(item, index) if on_start
|
133
|
+
|
129
134
|
begin
|
130
135
|
results[index] = call_with_index(items, index, options, &block)
|
131
136
|
rescue Exception => e
|
132
137
|
exception = e
|
133
138
|
break
|
139
|
+
ensure
|
140
|
+
on_finish.call(item, index) if on_finish
|
134
141
|
end
|
135
142
|
end
|
136
143
|
end
|
@@ -145,6 +152,8 @@ module Parallel
|
|
145
152
|
current_index = -1
|
146
153
|
results = []
|
147
154
|
exception = nil
|
155
|
+
on_start = options[:start]
|
156
|
+
on_finish = options[:finish]
|
148
157
|
|
149
158
|
in_threads(options[:count]) do |i|
|
150
159
|
worker = workers[i]
|
@@ -155,8 +164,13 @@ module Parallel
|
|
155
164
|
index = Thread.exclusive{ current_index += 1 }
|
156
165
|
break if index >= items.size
|
157
166
|
|
167
|
+
item = items[index]
|
168
|
+
|
158
169
|
Marshal.dump(index, worker[:write])
|
170
|
+
on_start.call(item, index) if on_start
|
171
|
+
|
159
172
|
output = Marshal.load(worker[:read])
|
173
|
+
on_finish.call(item, index) if on_finish
|
160
174
|
|
161
175
|
if ExceptionWrapper === output
|
162
176
|
exception = output.exception
|
data/lib/parallel/version.rb
CHANGED
data/spec/parallel_spec.rb
CHANGED
@@ -174,6 +174,38 @@ describe Parallel do
|
|
174
174
|
Process.should_not_receive(:fork)
|
175
175
|
Parallel.map([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
|
176
176
|
end
|
177
|
+
|
178
|
+
it "notifies when an item of work is dispatched to a worker process" do
|
179
|
+
monitor = double('monitor', :call => nil)
|
180
|
+
monitor.should_receive(:call).once.with(:first, 0)
|
181
|
+
monitor.should_receive(:call).once.with(:second, 1)
|
182
|
+
monitor.should_receive(:call).once.with(:third, 2)
|
183
|
+
Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 3) {}
|
184
|
+
end
|
185
|
+
|
186
|
+
it "notifies when an item of work is completed by a worker process" do
|
187
|
+
monitor = double('monitor', :call => nil)
|
188
|
+
monitor.should_receive(:call).once.with(:first, 0)
|
189
|
+
monitor.should_receive(:call).once.with(:second, 1)
|
190
|
+
monitor.should_receive(:call).once.with(:third, 2)
|
191
|
+
Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 3) {}
|
192
|
+
end
|
193
|
+
|
194
|
+
it "notifies when an item of work is dispatched to a threaded worker" do
|
195
|
+
monitor = double('monitor', :call => nil)
|
196
|
+
monitor.should_receive(:call).once.with(:first, 0)
|
197
|
+
monitor.should_receive(:call).once.with(:second, 1)
|
198
|
+
monitor.should_receive(:call).once.with(:third, 2)
|
199
|
+
Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 3) {}
|
200
|
+
end
|
201
|
+
|
202
|
+
it "notifies when an item of work is completed by a threaded worker" do
|
203
|
+
monitor = double('monitor', :call => nil)
|
204
|
+
monitor.should_receive(:call).once.with(:first, 0)
|
205
|
+
monitor.should_receive(:call).once.with(:second, 1)
|
206
|
+
monitor.should_receive(:call).once.with(:third, 2)
|
207
|
+
Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 3) {}
|
208
|
+
end
|
177
209
|
end
|
178
210
|
|
179
211
|
describe ".map_with_index" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.20
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-27 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description:
|
15
15
|
email: michael@grosser.it
|
@@ -66,7 +66,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
66
66
|
version: '0'
|
67
67
|
segments:
|
68
68
|
- 0
|
69
|
-
hash:
|
69
|
+
hash: 2227886358623378037
|
70
70
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
71
|
none: false
|
72
72
|
requirements:
|
@@ -75,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
75
75
|
version: '0'
|
76
76
|
segments:
|
77
77
|
- 0
|
78
|
-
hash:
|
78
|
+
hash: 2227886358623378037
|
79
79
|
requirements: []
|
80
80
|
rubyforge_project:
|
81
81
|
rubygems_version: 1.8.24
|